From 8d3d90a609dd40ba23ae06892635553940439533 Mon Sep 17 00:00:00 2001 From: Guillaume Deconinck Date: Thu, 19 Jun 2025 15:29:33 +0900 Subject: [PATCH 1/3] fix: correlate traces and logs together --- bins/nittei/src/telemetry.rs | 66 +++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/bins/nittei/src/telemetry.rs b/bins/nittei/src/telemetry.rs index 8b64a1ff..2217cdaa 100644 --- a/bins/nittei/src/telemetry.rs +++ b/bins/nittei/src/telemetry.rs @@ -1,4 +1,8 @@ -use opentelemetry::{global, propagation::TextMapCompositePropagator, trace::TracerProvider}; +use opentelemetry::{ + global, + propagation::TextMapCompositePropagator, + trace::{TraceContextExt, TracerProvider}, +}; use opentelemetry_datadog::{ApiVersion, DatadogPipelineBuilder, DatadogPropagator}; use opentelemetry_otlp::{WithExportConfig, WithHttpConfig}; use opentelemetry_sdk::{ @@ -6,8 +10,14 @@ use opentelemetry_sdk::{ propagation::TraceContextPropagator, trace::{self, RandomIdGenerator, Sampler, SdkTracerProvider}, }; -use tracing::warn; -use tracing_subscriber::{EnvFilter, Registry, layer::SubscriberExt}; +use tracing::{Event, Subscriber, warn}; +use tracing_subscriber::{ + EnvFilter, + Registry, + fmt::{FmtContext, FormatEvent, format::Writer}, + layer::SubscriberExt, + registry::LookupSpan, +}; /// Register a subscriber as global default to process span data. /// @@ -18,7 +28,7 @@ pub fn init_subscriber() -> anyhow::Result<()> { // If the binary is compiled in debug mode (aka for development) // use the compact format for logs - if cfg!(debug_assertions) { + if !cfg!(debug_assertions) { tracing_subscriber::fmt() .compact() .with_env_filter(env_filter) @@ -57,11 +67,7 @@ pub fn init_subscriber() -> anyhow::Result<()> { if let Some(telemetry_layer) = telemetry_layer { let subscriber = Registry::default() .with(env_filter) - .with( - tracing_subscriber::fmt::layer() - .json() - .with_current_span(false), - ) + .with(tracing_subscriber::fmt::layer().event_format(DatadogFormatter)) .with(telemetry_layer); // Set the global subscriber @@ -191,3 +197,45 @@ fn get_sampler() -> Sampler { // (2) if no parent, then the trace id ratio Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(ratio_to_sample))) } + +/// Empty struct to implement the FormatEvent trait +struct DatadogFormatter; + +/// Format the event to be compatible with Datadog +/// This allows to correlate the logs with the traces +impl FormatEvent for DatadogFormatter +where + S: Subscriber + for<'a> LookupSpan<'a>, + N: for<'a> tracing_subscriber::fmt::FormatFields<'a> + 'static, +{ + fn format_event( + &self, + ctx: &FmtContext<'_, S, N>, + mut writer: Writer<'_>, + event: &Event<'_>, + ) -> std::fmt::Result { + if let Some(span) = ctx.lookup_current() { + let extensions = span.extensions(); + let otel_ctx = extensions.get::(); + if let Some(ctx) = otel_ctx { + if let Some(span) = ctx + .span() + .span_context() + .is_valid() + .then(|| ctx.span().span_context().clone()) + { + write!( + writer, + "{{\"trace_id\":\"{}\",\"span_id\":\"{}\",", + span.trace_id(), + span.span_id() + )?; + } + } + } + + let json = tracing_subscriber::fmt::format::Format::default().json(); + + json.format_event(ctx, writer, event) + } +} From 40a95e2662fe6cbae64d79c150c5d3ea07cf0710 Mon Sep 17 00:00:00 2001 From: Guillaume Deconinck Date: Fri, 20 Jun 2025 10:42:15 +0900 Subject: [PATCH 2/3] fix: condition --- bins/nittei/src/telemetry.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bins/nittei/src/telemetry.rs b/bins/nittei/src/telemetry.rs index 2217cdaa..2b6f9f7a 100644 --- a/bins/nittei/src/telemetry.rs +++ b/bins/nittei/src/telemetry.rs @@ -28,7 +28,7 @@ pub fn init_subscriber() -> anyhow::Result<()> { // If the binary is compiled in debug mode (aka for development) // use the compact format for logs - if !cfg!(debug_assertions) { + if cfg!(debug_assertions) { tracing_subscriber::fmt() .compact() .with_env_filter(env_filter) From 040709cbda111a3e5fcfad44ba9990946a51b010 Mon Sep 17 00:00:00 2001 From: Guillaume Deconinck Date: Fri, 20 Jun 2025 11:14:51 +0900 Subject: [PATCH 3/3] fix: avoid having broken spans --- bins/nittei/src/telemetry.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/bins/nittei/src/telemetry.rs b/bins/nittei/src/telemetry.rs index 2b6f9f7a..9aa346b4 100644 --- a/bins/nittei/src/telemetry.rs +++ b/bins/nittei/src/telemetry.rs @@ -14,6 +14,7 @@ use tracing::{Event, Subscriber, warn}; use tracing_subscriber::{ EnvFilter, Registry, + field::VisitOutput, fmt::{FmtContext, FormatEvent, format::Writer}, layer::SubscriberExt, registry::LookupSpan, @@ -214,6 +215,10 @@ where mut writer: Writer<'_>, event: &Event<'_>, ) -> std::fmt::Result { + // Start the JSON object + write!(writer, "{{")?; + + // Add trace and span IDs if available if let Some(span) = ctx.lookup_current() { let extensions = span.extensions(); let otel_ctx = extensions.get::(); @@ -226,7 +231,7 @@ where { write!( writer, - "{{\"trace_id\":\"{}\",\"span_id\":\"{}\",", + "\"dd.trace_id\":\"{}\",\"dd.span_id\":\"{}\",", span.trace_id(), span.span_id() )?; @@ -234,8 +239,14 @@ where } } - let json = tracing_subscriber::fmt::format::Format::default().json(); + // Format the event fields + let mut visitor = tracing_subscriber::fmt::format::JsonVisitor::new(&mut writer); + event.record(&mut visitor); + visitor.finish()?; + + // Close the JSON object + write!(writer, "}}")?; - json.format_event(ctx, writer, event) + Ok(()) } }