From 5a205c8822ca16db400ef25a5d8b51a65c875669 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Tue, 12 May 2026 12:00:13 -0700 Subject: [PATCH 1/3] feat: use instrument_hooks markers in walltime --- .../instruments/hooks/__init__.py | 27 ++++++++++-- .../hooks/instrument_hooks_module.c | 1 + src/pytest_codspeed/instruments/walltime.py | 42 ++++++++++++++----- 3 files changed, 56 insertions(+), 14 deletions(-) diff --git a/src/pytest_codspeed/instruments/hooks/__init__.py b/src/pytest_codspeed/instruments/hooks/__init__.py index 983df8f..307e35a 100644 --- a/src/pytest_codspeed/instruments/hooks/__init__.py +++ b/src/pytest_codspeed/instruments/hooks/__init__.py @@ -13,9 +13,6 @@ if TYPE_CHECKING: from typing import Any, Callable -# Feature flags for instrument hooks -FEATURE_DISABLE_CALLGRIND_MARKERS = 0 - class InstrumentHooks: """Native library wrapper class providing benchmark measurement functionality.""" @@ -85,6 +82,30 @@ def set_executed_benchmark(self, uri: str, pid: int | None = None) -> None: if ret != 0: warnings.warn("Failed to set executed benchmark", RuntimeWarning) + @staticmethod + def current_timestamp() -> int: + """Return a monotonic timestamp in nanoseconds from the native library.""" + from . import dist_instrument_hooks # type: ignore + + return dist_instrument_hooks.instrument_hooks_current_timestamp() + + def add_marker( + self, marker_type: int, timestamp: int, pid: int | None = None + ) -> None: + """Emit a single marker at the given timestamp.""" + if pid is None: + pid = os.getpid() + ret = self._module.instrument_hooks_add_marker( + self._instance, pid, marker_type, timestamp + ) + if ret != 0: + warnings.warn("Failed to add marker", RuntimeWarning) + + def add_benchmark_timestamps(self, start: int, end: int) -> None: + """Emit a BenchmarkStart/BenchmarkEnd marker pair around a captured window.""" + self.add_marker(self._module.MARKER_TYPE_BENCHMARK_START, start) + self.add_marker(self._module.MARKER_TYPE_BENCHMARK_END, end) + def set_integration(self, name: str, version: str) -> None: """Set the integration name and version.""" ret = self._module.instrument_hooks_set_integration( diff --git a/src/pytest_codspeed/instruments/hooks/instrument_hooks_module.c b/src/pytest_codspeed/instruments/hooks/instrument_hooks_module.c index 1c1c409..ce205ea 100644 --- a/src/pytest_codspeed/instruments/hooks/instrument_hooks_module.c +++ b/src/pytest_codspeed/instruments/hooks/instrument_hooks_module.c @@ -296,6 +296,7 @@ PyMODINIT_FUNC PyInit_dist_instrument_hooks(void) { PyModule_AddIntConstant(module, "MARKER_TYPE_SAMPLE_END", MARKER_TYPE_SAMPLE_END); PyModule_AddIntConstant(module, "MARKER_TYPE_BENCHMARK_START", MARKER_TYPE_BENCHMARK_START); PyModule_AddIntConstant(module, "MARKER_TYPE_BENCHMARK_END", MARKER_TYPE_BENCHMARK_END); + PyModule_AddIntConstant(module, "FEATURE_DISABLE_CALLGRIND_MARKERS", FEATURE_DISABLE_CALLGRIND_MARKERS); #ifdef Py_GIL_DISABLED PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); diff --git a/src/pytest_codspeed/instruments/walltime.py b/src/pytest_codspeed/instruments/walltime.py index a23415b..c9d341c 100644 --- a/src/pytest_codspeed/instruments/walltime.py +++ b/src/pytest_codspeed/instruments/walltime.py @@ -183,7 +183,7 @@ def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]: ) return config_str, [] - def measure( + def measure( # noqa: C901 self, marker_options: BenchmarkMarkerOptions, name: str, @@ -232,21 +232,31 @@ def __codspeed_root_frame__() -> T: # Benchmark iter_range = range(iter_per_round) run_start = perf_counter_ns() - if self.instrument_hooks: - self.instrument_hooks.start_benchmark() + hooks = self.instrument_hooks + if hooks: + hooks.start_benchmark() for _ in range(rounds): + instrument_hooks_start = hooks.current_timestamp() if hooks else None start = perf_counter_ns() + for _ in iter_range: __codspeed_root_frame__() + end = perf_counter_ns() + if hooks and instrument_hooks_start is not None: + instrument_hooks_end = hooks.current_timestamp() + hooks.add_benchmark_timestamps( + instrument_hooks_start, instrument_hooks_end + ) + times_per_round_ns.append(end - start) if end - run_start > benchmark_config.max_time_ns: # TODO: log something break - if self.instrument_hooks: - self.instrument_hooks.stop_benchmark() - self.instrument_hooks.set_executed_benchmark(uri) + if hooks: + hooks.stop_benchmark() + hooks.set_executed_benchmark(uri) benchmark_end = perf_counter_ns() total_time = (benchmark_end - run_start) / 1e9 @@ -290,20 +300,30 @@ def __codspeed_root_frame__(*args, **kwargs) -> T: # Benchmark times_per_round_ns: list[float] = [] benchmark_start = perf_counter_ns() - if self.instrument_hooks: - self.instrument_hooks.start_benchmark() + hooks = self.instrument_hooks + if hooks: + hooks.start_benchmark() for _ in range(pedantic_options.rounds): args, kwargs = pedantic_options.setup_and_get_args_kwargs() + instrument_hooks_start = hooks.current_timestamp() if hooks else None start = perf_counter_ns() + for _ in iter_range: __codspeed_root_frame__(*args, **kwargs) + end = perf_counter_ns() + if hooks and instrument_hooks_start is not None: + instrument_hooks_end = hooks.current_timestamp() + hooks.add_benchmark_timestamps( + instrument_hooks_start, instrument_hooks_end + ) + times_per_round_ns.append(end - start) if pedantic_options.teardown is not None: pedantic_options.teardown(*args, **kwargs) - if self.instrument_hooks: - self.instrument_hooks.stop_benchmark() - self.instrument_hooks.set_executed_benchmark(uri) + if hooks: + hooks.stop_benchmark() + hooks.set_executed_benchmark(uri) benchmark_end = perf_counter_ns() total_time = (benchmark_end - benchmark_start) / 1e9 stats = BenchmarkStats.from_list( From ee98055fb69fba4cc62c3822ef3df4c7002352d3 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Tue, 12 May 2026 14:49:47 -0700 Subject: [PATCH 2/3] chore: use unsigned bash in the macos test --- .github/workflows/ci.yml | 2 ++ src/pytest_codspeed/instruments/analysis.py | 9 +++------ src/pytest_codspeed/instruments/hooks/__init__.py | 6 +++++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15b4569..f981ea9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,6 +93,8 @@ jobs: python-version: "3.14" - name: Install dependencies run: uv sync --all-extras --dev --locked + # Samply cannot profile signed binaries, this is a temporary workaround + - run: brew install bash - name: Run the benchmarks uses: CodSpeedHQ/action@main env: diff --git a/src/pytest_codspeed/instruments/analysis.py b/src/pytest_codspeed/instruments/analysis.py index 47cc1c8..5e75e91 100644 --- a/src/pytest_codspeed/instruments/analysis.py +++ b/src/pytest_codspeed/instruments/analysis.py @@ -6,10 +6,7 @@ from pytest_codspeed import __semver_version__ from pytest_codspeed.instruments import Instrument -from pytest_codspeed.instruments.hooks import ( - FEATURE_DISABLE_CALLGRIND_MARKERS, - InstrumentHooks, -) +from pytest_codspeed.instruments.hooks import InstrumentHooks from pytest_codspeed.utils import SUPPORTS_PERF_TRAMPOLINE if TYPE_CHECKING: @@ -79,7 +76,7 @@ def __codspeed_root_frame__() -> T: # Warmup CPython performance map cache __codspeed_root_frame__() - self.instrument_hooks.set_feature(FEATURE_DISABLE_CALLGRIND_MARKERS, True) + self.instrument_hooks.disable_callgrind_markers() self.instrument_hooks.start_benchmark() # Manually call the library function to avoid an extra stack frame. Also @@ -128,7 +125,7 @@ def __codspeed_root_frame__(*args, **kwargs) -> T: # Compute the actual result of the function args, kwargs = pedantic_options.setup_and_get_args_kwargs() - self.instrument_hooks.set_feature(FEATURE_DISABLE_CALLGRIND_MARKERS, True) + self.instrument_hooks.disable_callgrind_markers() self.instrument_hooks.start_benchmark() # Manually call the library function to avoid an extra stack frame. Also diff --git a/src/pytest_codspeed/instruments/hooks/__init__.py b/src/pytest_codspeed/instruments/hooks/__init__.py index 307e35a..2e6ffd5 100644 --- a/src/pytest_codspeed/instruments/hooks/__init__.py +++ b/src/pytest_codspeed/instruments/hooks/__init__.py @@ -118,7 +118,7 @@ def is_instrumented(self) -> bool: """Check if simulation is active.""" return self._module.instrument_hooks_is_instrumented(self._instance) - def set_feature(self, feature: int, enabled: bool) -> None: + def _set_feature(self, feature: int, enabled: bool) -> None: """Set a feature flag in the instrument hooks library. Args: @@ -127,6 +127,10 @@ def set_feature(self, feature: int, enabled: bool) -> None: """ self._module.instrument_hooks_set_feature(feature, enabled) + def disable_callgrind_markers(self, disabled: bool = True) -> None: + """Disable automatic callgrind markers around benchmark start/stop.""" + self._set_feature(self._module.FEATURE_DISABLE_CALLGRIND_MARKERS, disabled) + def set_environment(self, section_name: str, key: str, value: str) -> None: """Register a key-value pair under a named section for environment collection. From befdebfba2cdebddd84d6377f8d14d72e0ad4086 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Tue, 12 May 2026 15:35:33 -0700 Subject: [PATCH 3/3] chore: ignore common compilation warnings for instrument-hooks Transpiled code generates these. --- setup.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/setup.py b/setup.py index 63a9119..881cbbd 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,18 @@ "src/pytest_codspeed/instruments/hooks/instrument-hooks/dist/core.c", ], include_dirs=["src/pytest_codspeed/instruments/hooks/instrument-hooks/includes"], + # IMPORTANT: Keep in sync with instrument-hooks/.github/workflows/ci.yml + # (COMMON_CFLAGS). The Zig-generated core.c emits many warnings that + # upstream silences; in particular distros like Nix/Debian/Fedora inject + # -Werror=format-security, which would otherwise fail the build. + extra_compile_args=[ + "-Wno-format", + "-Wno-format-security", + "-Wno-unused-but-set-variable", + "-Wno-unused-const-variable", + "-Wno-type-limits", + "-Wno-uninitialized", + ], optional=not IS_EXTENSION_REQUIRED, )