From b3b9451222eb2a4466d9bc6d7d455b802a888a28 Mon Sep 17 00:00:00 2001 From: zhaozx-cn Date: Fri, 12 Jun 2026 22:28:35 +0800 Subject: [PATCH 1/5] add device print Signed-off-by: zhaozx-cn --- .../sglang/srt/hardware_backend/npu/utils.py | 70 ++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/hardware_backend/npu/utils.py b/python/sglang/srt/hardware_backend/npu/utils.py index 5ab80ee38b1e..0424ee21675c 100644 --- a/python/sglang/srt/hardware_backend/npu/utils.py +++ b/python/sglang/srt/hardware_backend/npu/utils.py @@ -2,7 +2,7 @@ import logging import sys from enum import IntEnum -from typing import TYPE_CHECKING, Callable +from typing import TYPE_CHECKING, Any, Callable import torch @@ -339,3 +339,71 @@ def process_routed_expert(hidden_states, topk_output, forward_func): with torch.get_device_module().stream(stream): shared_output = forward_func(hidden_states, topk_output) return shared_output + + +def _mark_op_side_effectful(op: Any) -> None: + torch.fx.node.has_side_effect(op) + default_overload = getattr(op, "default", None) + if default_overload is not None: + torch.fx.node.has_side_effect(default_overload) + + +def _ensure_device_print_registered() -> None: + try: + # Mark device_print ops side-effectful so FX/Inductor does not DCE or reorder these debug callbacks. + import sgl_kernel_npu + _mark_op_side_effectful(torch.ops.npu.device_print) + _mark_op_side_effectful(torch.ops.npu.device_print_tensor) + except AttributeError as exc: + raise RuntimeError( + "device_print requires _C_ascend.device_print ops to be available " + "when custom ops are enabled in the current Ascend build." + ) from exc + + +def device_print( + value: torch.Tensor | int | float | bool | str | torch.dtype | torch.device | torch.Size, +) -> None: + """Print one value from a device callback. + + This helper is intended for debugging. To stay replay-safe under + ``torch.npu.graph`` capture/replay, the underlying callback payloads are + retained instead of being reclaimed after the first host callback runs. + Avoid using it in hot paths or long-running high-frequency loops, otherwise + there may be memory issues due to too many retained payloads. + + Supported usage: + + >>> from sglang.srt.hardware_backend.npu.utils import device_print + >>> device_print(x) + >>> device_print("already formatted text") + >>> device_print(7) + + Unsupported usage: + + >>> device_print("x =", x) + >>> device_print("This is ", x, "and this is ", y) + + If you need device-time tensor values, pass the tensor itself. If you need + text, pass one final string that is already formatted. + + Tensor values are copied to host on the current stream before the callback + prints them, so printing remains ordered with respect to the surrounding + device work. + + DO NOT FORMAT A DEVICE TENSOR INTO A STRING YOURSELF AND THEN PRINT, for example: + + >>> device_print(f"x = {x}") + >>> device_print("x = " + str(x)) + """ + _ensure_device_print_registered() + + if isinstance(value, torch.Tensor): + torch.ops.npu.device_print_tensor(value) + elif isinstance(value, (str, int, float, bool, torch.dtype, torch.device, torch.Size)): + torch.ops.npu.device_print(str(value)) + else: + raise TypeError( + f"Unsupported device_print value type: {type(value)!r}. " + "Use exactly one argument: device_print(tensor), device_print('formatted text')." + ) From 0394cb9ecc0eec1bb873b911ea85c9bd4608d3be Mon Sep 17 00:00:00 2001 From: zhaozx-cn Date: Thu, 18 Jun 2026 14:35:24 +0800 Subject: [PATCH 2/5] fix lint Signed-off-by: zhaozx-cn --- 1 | 48 +++++++++++++++++++ print.patch | 48 +++++++++++++++++++ .../sglang/srt/hardware_backend/npu/utils.py | 22 +++++++-- 3 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 1 create mode 100644 print.patch diff --git a/1 b/1 new file mode 100644 index 000000000000..b2b88b307a2b --- /dev/null +++ b/1 @@ -0,0 +1,48 @@ +diff --git a/python/sglang/srt/hardware_backend/npu/utils.py b/python/sglang/srt/hardware_backend/npu/utils.py +index 0424ee216..7a6bd83c6 100644 +--- a/python/sglang/srt/hardware_backend/npu/utils.py ++++ b/python/sglang/srt/hardware_backend/npu/utils.py +@@ -351,18 +351,28 @@ def _mark_op_side_effectful(op: Any) -> None: + def _ensure_device_print_registered() -> None: + try: + # Mark device_print ops side-effectful so FX/Inductor does not DCE or reorder these debug callbacks. +- import sgl_kernel_npu ++ import sgl_kernel_npu # noqa: F401 ++ + _mark_op_side_effectful(torch.ops.npu.device_print) + _mark_op_side_effectful(torch.ops.npu.device_print_tensor) + except AttributeError as exc: + raise RuntimeError( +- "device_print requires _C_ascend.device_print ops to be available " +- "when custom ops are enabled in the current Ascend build." ++ "device_print ops are available in sgl_kernel_npu. " ++ "Please install the latest sgl_kernel_npu from source." + ) from exc + + + def device_print( +- value: torch.Tensor | int | float | bool | str | torch.dtype | torch.device | torch.Size, ++ value: ( ++ torch.Tensor ++ | int ++ | float ++ | bool ++ | str ++ | torch.dtype ++ | torch.device ++ | torch.Size ++ ), + ) -> None: + """Print one value from a device callback. + +@@ -400,7 +410,9 @@ def device_print( + + if isinstance(value, torch.Tensor): + torch.ops.npu.device_print_tensor(value) +- elif isinstance(value, (str, int, float, bool, torch.dtype, torch.device, torch.Size)): ++ elif isinstance( ++ value, (str, int, float, bool, torch.dtype, torch.device, torch.Size) ++ ): + torch.ops.npu.device_print(str(value)) + else: + raise TypeError( diff --git a/print.patch b/print.patch new file mode 100644 index 000000000000..b2b88b307a2b --- /dev/null +++ b/print.patch @@ -0,0 +1,48 @@ +diff --git a/python/sglang/srt/hardware_backend/npu/utils.py b/python/sglang/srt/hardware_backend/npu/utils.py +index 0424ee216..7a6bd83c6 100644 +--- a/python/sglang/srt/hardware_backend/npu/utils.py ++++ b/python/sglang/srt/hardware_backend/npu/utils.py +@@ -351,18 +351,28 @@ def _mark_op_side_effectful(op: Any) -> None: + def _ensure_device_print_registered() -> None: + try: + # Mark device_print ops side-effectful so FX/Inductor does not DCE or reorder these debug callbacks. +- import sgl_kernel_npu ++ import sgl_kernel_npu # noqa: F401 ++ + _mark_op_side_effectful(torch.ops.npu.device_print) + _mark_op_side_effectful(torch.ops.npu.device_print_tensor) + except AttributeError as exc: + raise RuntimeError( +- "device_print requires _C_ascend.device_print ops to be available " +- "when custom ops are enabled in the current Ascend build." ++ "device_print ops are available in sgl_kernel_npu. " ++ "Please install the latest sgl_kernel_npu from source." + ) from exc + + + def device_print( +- value: torch.Tensor | int | float | bool | str | torch.dtype | torch.device | torch.Size, ++ value: ( ++ torch.Tensor ++ | int ++ | float ++ | bool ++ | str ++ | torch.dtype ++ | torch.device ++ | torch.Size ++ ), + ) -> None: + """Print one value from a device callback. + +@@ -400,7 +410,9 @@ def device_print( + + if isinstance(value, torch.Tensor): + torch.ops.npu.device_print_tensor(value) +- elif isinstance(value, (str, int, float, bool, torch.dtype, torch.device, torch.Size)): ++ elif isinstance( ++ value, (str, int, float, bool, torch.dtype, torch.device, torch.Size) ++ ): + torch.ops.npu.device_print(str(value)) + else: + raise TypeError( diff --git a/python/sglang/srt/hardware_backend/npu/utils.py b/python/sglang/srt/hardware_backend/npu/utils.py index 0424ee21675c..7a6bd83c6858 100644 --- a/python/sglang/srt/hardware_backend/npu/utils.py +++ b/python/sglang/srt/hardware_backend/npu/utils.py @@ -351,18 +351,28 @@ def _mark_op_side_effectful(op: Any) -> None: def _ensure_device_print_registered() -> None: try: # Mark device_print ops side-effectful so FX/Inductor does not DCE or reorder these debug callbacks. - import sgl_kernel_npu + import sgl_kernel_npu # noqa: F401 + _mark_op_side_effectful(torch.ops.npu.device_print) _mark_op_side_effectful(torch.ops.npu.device_print_tensor) except AttributeError as exc: raise RuntimeError( - "device_print requires _C_ascend.device_print ops to be available " - "when custom ops are enabled in the current Ascend build." + "device_print ops are available in sgl_kernel_npu. " + "Please install the latest sgl_kernel_npu from source." ) from exc def device_print( - value: torch.Tensor | int | float | bool | str | torch.dtype | torch.device | torch.Size, + value: ( + torch.Tensor + | int + | float + | bool + | str + | torch.dtype + | torch.device + | torch.Size + ), ) -> None: """Print one value from a device callback. @@ -400,7 +410,9 @@ def device_print( if isinstance(value, torch.Tensor): torch.ops.npu.device_print_tensor(value) - elif isinstance(value, (str, int, float, bool, torch.dtype, torch.device, torch.Size)): + elif isinstance( + value, (str, int, float, bool, torch.dtype, torch.device, torch.Size) + ): torch.ops.npu.device_print(str(value)) else: raise TypeError( From 4d7f1f81835379d51d1282dfb0df44c3f899b27a Mon Sep 17 00:00:00 2001 From: zhaozx-cn Date: Thu, 18 Jun 2026 14:42:09 +0800 Subject: [PATCH 3/5] fix lint Signed-off-by: zhaozx-cn --- print.patch | 48 ------------------------------------------------ 1 file changed, 48 deletions(-) delete mode 100644 print.patch diff --git a/print.patch b/print.patch deleted file mode 100644 index b2b88b307a2b..000000000000 --- a/print.patch +++ /dev/null @@ -1,48 +0,0 @@ -diff --git a/python/sglang/srt/hardware_backend/npu/utils.py b/python/sglang/srt/hardware_backend/npu/utils.py -index 0424ee216..7a6bd83c6 100644 ---- a/python/sglang/srt/hardware_backend/npu/utils.py -+++ b/python/sglang/srt/hardware_backend/npu/utils.py -@@ -351,18 +351,28 @@ def _mark_op_side_effectful(op: Any) -> None: - def _ensure_device_print_registered() -> None: - try: - # Mark device_print ops side-effectful so FX/Inductor does not DCE or reorder these debug callbacks. -- import sgl_kernel_npu -+ import sgl_kernel_npu # noqa: F401 -+ - _mark_op_side_effectful(torch.ops.npu.device_print) - _mark_op_side_effectful(torch.ops.npu.device_print_tensor) - except AttributeError as exc: - raise RuntimeError( -- "device_print requires _C_ascend.device_print ops to be available " -- "when custom ops are enabled in the current Ascend build." -+ "device_print ops are available in sgl_kernel_npu. " -+ "Please install the latest sgl_kernel_npu from source." - ) from exc - - - def device_print( -- value: torch.Tensor | int | float | bool | str | torch.dtype | torch.device | torch.Size, -+ value: ( -+ torch.Tensor -+ | int -+ | float -+ | bool -+ | str -+ | torch.dtype -+ | torch.device -+ | torch.Size -+ ), - ) -> None: - """Print one value from a device callback. - -@@ -400,7 +410,9 @@ def device_print( - - if isinstance(value, torch.Tensor): - torch.ops.npu.device_print_tensor(value) -- elif isinstance(value, (str, int, float, bool, torch.dtype, torch.device, torch.Size)): -+ elif isinstance( -+ value, (str, int, float, bool, torch.dtype, torch.device, torch.Size) -+ ): - torch.ops.npu.device_print(str(value)) - else: - raise TypeError( From 081e4ccf99e248c2889c616b8de09efbd138fd33 Mon Sep 17 00:00:00 2001 From: zhaozx-cn Date: Thu, 18 Jun 2026 14:42:56 +0800 Subject: [PATCH 4/5] fix lint Signed-off-by: zhaozx-cn --- 1 | 48 ------------------------------------------------ 1 file changed, 48 deletions(-) delete mode 100644 1 diff --git a/1 b/1 deleted file mode 100644 index b2b88b307a2b..000000000000 --- a/1 +++ /dev/null @@ -1,48 +0,0 @@ -diff --git a/python/sglang/srt/hardware_backend/npu/utils.py b/python/sglang/srt/hardware_backend/npu/utils.py -index 0424ee216..7a6bd83c6 100644 ---- a/python/sglang/srt/hardware_backend/npu/utils.py -+++ b/python/sglang/srt/hardware_backend/npu/utils.py -@@ -351,18 +351,28 @@ def _mark_op_side_effectful(op: Any) -> None: - def _ensure_device_print_registered() -> None: - try: - # Mark device_print ops side-effectful so FX/Inductor does not DCE or reorder these debug callbacks. -- import sgl_kernel_npu -+ import sgl_kernel_npu # noqa: F401 -+ - _mark_op_side_effectful(torch.ops.npu.device_print) - _mark_op_side_effectful(torch.ops.npu.device_print_tensor) - except AttributeError as exc: - raise RuntimeError( -- "device_print requires _C_ascend.device_print ops to be available " -- "when custom ops are enabled in the current Ascend build." -+ "device_print ops are available in sgl_kernel_npu. " -+ "Please install the latest sgl_kernel_npu from source." - ) from exc - - - def device_print( -- value: torch.Tensor | int | float | bool | str | torch.dtype | torch.device | torch.Size, -+ value: ( -+ torch.Tensor -+ | int -+ | float -+ | bool -+ | str -+ | torch.dtype -+ | torch.device -+ | torch.Size -+ ), - ) -> None: - """Print one value from a device callback. - -@@ -400,7 +410,9 @@ def device_print( - - if isinstance(value, torch.Tensor): - torch.ops.npu.device_print_tensor(value) -- elif isinstance(value, (str, int, float, bool, torch.dtype, torch.device, torch.Size)): -+ elif isinstance( -+ value, (str, int, float, bool, torch.dtype, torch.device, torch.Size) -+ ): - torch.ops.npu.device_print(str(value)) - else: - raise TypeError( From 6fa0dac6111eebccb67bdfbfe9e73c82ad7a2e15 Mon Sep 17 00:00:00 2001 From: zhaozx-cn Date: Mon, 22 Jun 2026 09:48:55 +0800 Subject: [PATCH 5/5] fix lint Signed-off-by: zhaozx-cn --- python/sglang/srt/hardware_backend/npu/utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/sglang/srt/hardware_backend/npu/utils.py b/python/sglang/srt/hardware_backend/npu/utils.py index 7a6bd83c6858..534f9d03ab19 100644 --- a/python/sglang/srt/hardware_backend/npu/utils.py +++ b/python/sglang/srt/hardware_backend/npu/utils.py @@ -16,6 +16,7 @@ _is_npu = is_npu() indexer_weight_stream = None gva_is_inited = False +device_print_registered = False class NPUACLFormat(IntEnum): @@ -349,12 +350,16 @@ def _mark_op_side_effectful(op: Any) -> None: def _ensure_device_print_registered() -> None: + global device_print_registered + if device_print_registered: + return try: # Mark device_print ops side-effectful so FX/Inductor does not DCE or reorder these debug callbacks. import sgl_kernel_npu # noqa: F401 _mark_op_side_effectful(torch.ops.npu.device_print) _mark_op_side_effectful(torch.ops.npu.device_print_tensor) + device_print_registered = True except AttributeError as exc: raise RuntimeError( "device_print ops are available in sgl_kernel_npu. "