From 16b26d3fc7e1cafd7db82a8d1b4d562196974e6c Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 14 Apr 2026 18:17:02 +0200 Subject: [PATCH 1/2] fix(walltime): also capture kernel callstacks --- src/executor/wall_time/perf/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/executor/wall_time/perf/mod.rs b/src/executor/wall_time/perf/mod.rs index c854d49a..1bf35104 100644 --- a/src/executor/wall_time/perf/mod.rs +++ b/src/executor/wall_time/perf/mod.rs @@ -151,7 +151,6 @@ impl PerfRunner { "--freq=997", // Use a prime number to avoid synchronization with periodic tasks "--delay=-1", "-g", - "--user-callchains", &format!("--call-graph={cg_mode}"), &format!( "--control=fifo:{},{}", From f2276dadd0a551fe69b2eb764de30370793d1419 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 14 Apr 2026 18:49:36 +0200 Subject: [PATCH 2/2] feat(walltime): dump kallsyms symbols --- src/executor/wall_time/perf/kallsyms.rs | 139 ++++++++++++++++++++++++ src/executor/wall_time/perf/mod.rs | 10 +- 2 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 src/executor/wall_time/perf/kallsyms.rs diff --git a/src/executor/wall_time/perf/kallsyms.rs b/src/executor/wall_time/perf/kallsyms.rs new file mode 100644 index 00000000..4710a8ce --- /dev/null +++ b/src/executor/wall_time/perf/kallsyms.rs @@ -0,0 +1,139 @@ +use crate::executor::wall_time::perf::module_symbols::{ModuleSymbols, Symbol}; +use crate::prelude::*; +use libc::pid_t; +use std::collections::HashSet; +use std::path::Path; + +const KALLSYMS_PATH: &str = "/proc/kallsyms"; + +pub fn dump_kallsyms_as_perf_map_for_pids( + profile_folder: &Path, + pids: &HashSet, +) -> Result<()> { + let content = std::fs::read_to_string(KALLSYMS_PATH)?; + append_kallsyms_content_to_perf_maps_for_pids(profile_folder, pids, &content) +} + +fn append_kallsyms_content_to_perf_maps_for_pids( + profile_folder: &Path, + pids: &HashSet, + content: &str, +) -> Result<()> { + let symbols = parse_kallsyms_content(content); + if symbols.is_empty() { + return Ok(()); + } + let symbols = ModuleSymbols::new(symbols); + + for pid in pids { + symbols.append_to_file(profile_folder.join(format!("perf-{pid}.map")))?; + } + + Ok(()) +} + +fn parse_kallsyms_content(content: &str) -> Vec { + let mut raw_symbols = content + .lines() + .filter_map(parse_kallsyms_line) + .collect::>(); + raw_symbols.sort_by_key(|(addr, _)| *addr); + + let mut symbols = Vec::with_capacity(raw_symbols.len()); + for index in 0..raw_symbols.len() { + let (addr, name) = &raw_symbols[index]; + + let mut next_index = index + 1; + while next_index < raw_symbols.len() && raw_symbols[next_index].0 <= *addr { + next_index += 1; + } + + let size = if let Some((next_addr, _)) = raw_symbols.get(next_index) { + next_addr - addr + } else { + const LAST_SYMBOL_SIZE: u64 = 4096; + LAST_SYMBOL_SIZE + }; + + symbols.push(Symbol { + addr: *addr, + size, + name: name.clone(), + }); + } + + symbols +} + +fn parse_kallsyms_line(line: &str) -> Option<(u64, String)> { + let mut parts = line.split_whitespace(); + let addr = u64::from_str_radix(parts.next()?, 16).ok()?; + if addr == 0 { + return None; + } + + let _symbol_type = parts.next()?; + let name = parts.next()?.to_string(); + Some((addr, name)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_and_sizes_kallsyms_symbols() { + let content = r#" +ffffffff81000000 T _stext +ffffffff81000100 t secondary_startup_64 +0000000000000000 t should_be_skipped +ffffffff81000300 T cpu_startup_entry +"#; + + let parsed = parse_kallsyms_content(content); + + assert_eq!(parsed.len(), 3); + assert_eq!(parsed[0].addr, 0xffffffff81000000); + assert_eq!(parsed[0].size, 0x100); + assert_eq!(parsed[0].name, "_stext"); + + assert_eq!(parsed[1].addr, 0xffffffff81000100); + assert_eq!(parsed[1].size, 0x200); + assert_eq!(parsed[1].name, "secondary_startup_64"); + + assert_eq!(parsed[2].addr, 0xffffffff81000300); + assert_eq!(parsed[2].size, 0x1000); + assert_eq!(parsed[2].name, "cpu_startup_entry"); + } + + #[test] + fn appends_kallsyms_to_each_pid_perf_map() { + let dir = tempfile::tempdir().unwrap(); + let profile_folder = dir.path(); + + let mut pids = HashSet::new(); + pids.insert(1234); + pids.insert(5678); + + std::fs::write(profile_folder.join("perf-1234.map"), "existing\n").unwrap(); + + let content = r#" +ffffffff81000000 T _stext +ffffffff81000100 t secondary_startup_64 +"#; + + append_kallsyms_content_to_perf_maps_for_pids(profile_folder, &pids, content).unwrap(); + + let first = std::fs::read_to_string(profile_folder.join("perf-1234.map")).unwrap(); + assert_eq!( + first, + "existing\nffffffff81000000 100 _stext\nffffffff81000100 1000 secondary_startup_64\n" + ); + + let second = std::fs::read_to_string(profile_folder.join("perf-5678.map")).unwrap(); + assert_eq!( + second, + "ffffffff81000000 100 _stext\nffffffff81000100 1000 secondary_startup_64\n" + ); + } +} diff --git a/src/executor/wall_time/perf/mod.rs b/src/executor/wall_time/perf/mod.rs index 1bf35104..46b6b823 100644 --- a/src/executor/wall_time/perf/mod.rs +++ b/src/executor/wall_time/perf/mod.rs @@ -29,6 +29,7 @@ use std::path::PathBuf; use std::{cell::OnceCell, process::ExitStatus}; mod jit_dump; +mod kallsyms; mod naming; mod parse_perf_file; mod save_artifacts; @@ -308,14 +309,19 @@ impl BenchmarkData { // Harvest the perf maps generated by python. This will copy the perf // maps from /tmp to the profile folder. We have to write our own perf - // maps to these files AFTERWARDS, otherwise it'll be overwritten! - debug!("Harvesting perf maps and jit dumps for pids: {tracked_pids:?}"); + // maps (kallsyms / jit dump) afterwards, otherwise they'll be overwritten! + debug!("Harvesting perf maps for pids: {tracked_pids:?}"); harvest_perf_maps_for_pids(path_ref, &tracked_pids) .await .map_err(|e| { error!("Failed to harvest perf maps: {e}"); BenchmarkDataSaveError::FailedToHarvestPerfMaps })?; + + if let Err(error) = kallsyms::dump_kallsyms_as_perf_map_for_pids(path_ref, &tracked_pids) { + warn!("Failed to dump kernel symbols from /proc/kallsyms: {error}"); + } + let jit_unwind_data_by_pid = jit_dump::save_symbols_and_harvest_unwind_data_for_pids(path_ref, &tracked_pids) .await