Skip to content

Add E2E wall-clock timing for store requests (GPU IPC + async data paths) #373

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Copilot wants to merge 3 commits into
base: copilot/ww24-pr-async-again
Choose a base branch
from
+16 −0
Draft

Add E2E wall-clock timing for store requests (GPU IPC + async data paths) #373

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
880c0fe
Initial plan
Copilot Jun 17, 2026
84ac5eb
Add E2E timing from submit_store_request to get_finished
Copilot Jun 17, 2026
0d993fc
Remove redundant str() in E2E-STORE log call
Copilot Jun 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions lmcache/integration/vllm/vllm_multi_process_adapter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import enum
import os
import threading
import time

# Third Party
import torch
Expand Down Expand Up @@ -936,6 +937,10 @@ def __init__(
# Prevents re-reporting the same ID after drain clears tracking sets.
self._returned_finished: set[str] = set()

# Timestamps recorded when submit_store_request is called, used to
# measure E2E wall-clock time until the future is resolved.
self._store_submit_times: dict[str, float] = {}

self.model_name = model_name
self.parallel_strategy = parallel_strategy

Expand Down Expand Up @@ -1185,6 +1190,7 @@ def submit_store_request(
)
self.store_futures[request_id] = future
self.store_events[request_id] = event
self._store_submit_times[request_id] = time.perf_counter()

@_lmcache_nvtx_annotate
def submit_retrieve_request(
Expand Down Expand Up @@ -1345,6 +1351,7 @@ def get_finished(
self.retrieve_futures.clear()
self.store_events.clear()
self.retrieve_events.clear()
self._store_submit_times.clear()

ret_stores = self._process_finished_stores(
finished_stores, finished_req_ids_from_engine
Expand All @@ -1363,6 +1370,15 @@ def get_finished(
if not s_future.query():
continue

_t_done = time.perf_counter()
_t_submit = self._store_submit_times.pop(request_id, None)
if _t_submit is not None:
logger.info(
"[E2E-STORE] req=%s e2e=%.3f ms",
request_id,
(_t_done - _t_submit) * 1000,
)

s_result = s_future.result()
finished_stores.add(request_id)

Expand Down