Skip to content

[CLN] Clean up OTel config for better tracing #4578

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

[CLN] Clean up OTel config for better tracing #4578

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1d0f1a7
[CLN] Clean up OTel config for better tracing
drewkim May 19, 2025
a00c345
rm stdout layers
drewkim May 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 9 additions & 42 deletions rust/tracing/src/init_tracer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ pub fn init_global_filter_layer() -> Box<dyn Layer<Registry> + Send + Sync> {
"compaction_service",
"distance_metrics",
"full_text",
"hosted-frontend",
"hosted_frontend",
"billing_service",
"metadata_filtering",
"query_service",
"wal3",
Expand Down Expand Up @@ -107,53 +108,19 @@ pub fn init_otel_layer(
.with_resource(resource.clone())
.build();
global::set_meter_provider(meter_provider);
// Layer for adding our configured tracer.
// Export everything at this layer. The backend i.e. honeycomb or jaeger will filter at its end.
tracing_opentelemetry::OpenTelemetryLayer::new(tracer)
.with_filter(tracing_subscriber::filter::LevelFilter::TRACE)
.boxed()
tracing_opentelemetry::OpenTelemetryLayer::new(tracer).boxed()
}

pub fn init_stdout_layer() -> Box<dyn Layer<Registry> + Send + Sync> {
fmt::layer()
.pretty()
.with_target(false)
.with_filter(tracing_subscriber::filter::FilterFn::new(|metadata| {
// NOTE(rescrv): This is a hack, too. Not an uppercase hack, just a hack. This
// one's localized to the cache module. There's not much to do to unify it with
// the otel filter because these are different output layers from the tracing.

// This filter ensures that we don't cache calls for get/insert on stdout, but will
// still see the clear call.
!(metadata
.module_path()
.unwrap_or("")
.starts_with("chroma_cache")
&& metadata.name() != "clear")
}))
.with_filter(tracing_subscriber::filter::FilterFn::new(|metadata| {
metadata.module_path().unwrap_or("").starts_with("chroma")
|| metadata.module_path().unwrap_or("").starts_with("wal3")
|| metadata.module_path().unwrap_or("").starts_with("worker")
|| metadata
.module_path()
.unwrap_or("")
.starts_with("garbage_collector")
|| metadata
.module_path()
.unwrap_or("")
.starts_with("opentelemetry_sdk")
|| metadata
.module_path()
.unwrap_or("")
.starts_with("hosted-frontend")
}))
.with_filter(tracing_subscriber::filter::LevelFilter::INFO)
.boxed()
fmt::layer().pretty().with_target(false).boxed()
}

pub fn init_tracing(layers: Vec<Box<dyn Layer<Registry> + Send + Sync>>) {
global::set_text_map_propagator(TraceContextPropagator::new());
let layers = layers
.into_iter()
.reduce(|a, b| Box::new(a.and_then(b)))
.expect("Should be able to create tracing layers");
let subscriber = tracing_subscriber::registry().with(layers);
tracing::subscriber::set_global_default(subscriber)
.expect("Should be able to set global tracing subscriber");
Expand Down Expand Up @@ -184,7 +151,7 @@ pub fn init_panic_tracing_hook() {

pub fn init_otel_tracing(service_name: &String, otel_endpoint: &String) {
let layers = vec![
init_global_filter_layer(),
// init_global_filter_layer(),
init_otel_layer(service_name, otel_endpoint),
init_stdout_layer(),
];
Expand Down
Loading