VTK interactor doesn't register mouse press/release events when another button is held

[sudomakeinstall]

Describe the bug

In pure VTK, mouse press/release events are triggered regardless of whether another mouse button is currently pressed. For example, a "LeftButtonPressEvent", "RightButtonPressEvent", "LeftButtonReleaseEvent", and "RightButtonReleaseEvent" can all be recorded in sequence. However, when using a VTK render window within Trame, button press/release events are not triggered if another mouse button is currently pressed. In the above example, the "RightButtonPressEvent" is not triggered (since the left button is pressed), the "LeftButtonReleaseEvent" is not triggered (since the right button is pressed), and the "RightButtonReleaseEvent" is not triggered (presumably since the right press was never registered).

To Reproduce

1. I'm using a freshly created uv environment; version bounds from my pyproject.toml as follows:

dependencies = [
    "trame-vtk>=2.9.1",
    "trame-vuetify>=3.0.3",
    "trame>=3.12.0",
    "vtk>=9.5.1",
]

2. Run the script copied below passing "trame" as the argument:

./test_multibutton_comparison.py trame

3. In the VTK render window that opens within the browser, press the left button, drag, press the right button, drag, release the left button, drag, release the right button, drag. You will get an output similar to the following, with the mouse left in the "drag" state, though no buttons are depressed:

LEFT BUTTON PRESS
DRAG: L=True R=False delta=[0,0]
DRAG: L=True R=False delta=[113,237]
DRAG: L=True R=False delta=[0,0]
DRAG: L=True R=False delta=[-1,0]
DRAG: L=True R=False delta=[-2,0]
DRAG: L=True R=False delta=[-1,-1]
DRAG: L=True R=False delta=[0,0]
DRAG: L=True R=False delta=[-1,0]
DRAG: L=True R=False delta=[-3,-1]
DRAG: L=True R=False delta=[-9,-8]
DRAG: L=True R=False delta=[-1,-2]
DRAG: L=True R=False delta=[-2,-4]
DRAG: L=True R=False delta=[-2,-3]

In order to "release" the mouse, you need to press and release the left mouse button again.

Code

#!/usr/bin/env python3
"""
Multi-button test comparison: VTK vs Trame
Usage:
    ./test_multibutton_comparison.py vtk    # Native VTK (multi-button works)
    ./test_multibutton_comparison.py trame  # Trame/web (multi-button blocked)
"""

import sys
import vtk


class MultiButtonInteractor(vtk.vtkInteractorStyle):
    def __init__(self):
        super().__init__()
        self.left_button_down = False
        self.right_button_down = False
        self.last_mouse_pos = [0, 0]

        self.AddObserver("LeftButtonPressEvent", self.on_left_press)
        self.AddObserver("LeftButtonReleaseEvent", self.on_left_release)
        self.AddObserver("RightButtonPressEvent", self.on_right_press)
        self.AddObserver("RightButtonReleaseEvent", self.on_right_release)
        self.AddObserver("MouseMoveEvent", self.on_mouse_move)

    def on_left_press(self, obj, event):
        print("LEFT BUTTON PRESS")
        self.left_button_down = True
        interactor = self.GetInteractor()
        if interactor:
            self.last_mouse_pos = list(interactor.GetEventPosition())

    def on_left_release(self, obj, event):
        print("LEFT BUTTON RELEASE")
        self.left_button_down = False

    def on_right_press(self, obj, event):
        print("RIGHT BUTTON PRESS")
        self.right_button_down = True
        interactor = self.GetInteractor()
        if interactor:
            self.last_mouse_pos = list(interactor.GetEventPosition())

    def on_right_release(self, obj, event):
        print("RIGHT BUTTON RELEASE")
        self.right_button_down = False

    def on_mouse_move(self, obj, event):
        if self.left_button_down or self.right_button_down:
            interactor = self.GetInteractor()
            if interactor:
                current_pos = list(interactor.GetEventPosition())
                dx = current_pos[0] - self.last_mouse_pos[0]
                dy = current_pos[1] - self.last_mouse_pos[1]
                self.last_mouse_pos = current_pos

                print(
                    f"DRAG: L={self.left_button_down} R={self.right_button_down} delta=[{dx},{dy}]"
                )

                if self.left_button_down and self.right_button_down:
                    print("  *** LEFT+RIGHT COMBINATION DETECTED! ***")


def create_vtk_scene():
    """Shared VTK scene setup"""
    renderer = vtk.vtkRenderer()
    render_window = vtk.vtkRenderWindow()
    render_window.AddRenderer(renderer)
    
    interactor = vtk.vtkRenderWindowInteractor()
    interactor.SetRenderWindow(render_window)
    style = MultiButtonInteractor()
    interactor.SetInteractorStyle(style)
    
    return renderer, render_window, interactor


def run_mode(mode):
    if mode == "vtk":
        print("=== VTK MODE (Native) ===")
        print("Multi-button combinations SHOULD work")
        print("Try: Left-drag, then add Right button while dragging")
        print("")
        
        renderer, render_window, interactor = create_vtk_scene()
        render_window.SetWindowName("VTK Mode - Multi-button should work")
        
        interactor.Initialize()
        render_window.Render()
        interactor.Start()
        
    elif mode == "trame":
        print("=== TRAME MODE (Web) ===")
        print("Multi-button combinations should be BLOCKED")
        print("Try: Left-drag, then add Right button while dragging")
        print("Opening browser...")
        print("")
        
        from trame.app import get_server
        from trame.ui.vuetify3 import SinglePageLayout
        from trame.widgets import vtk as vtk_widgets
        
        server = get_server()
        
        renderer, render_window, interactor = create_vtk_scene()
        render_window.SetOffScreenRendering(True)
        
        with SinglePageLayout(server) as layout:
            layout.title.set_text("Trame Mode - Multi-button should be blocked")
            with layout.content:
                vtk_widgets.VtkRemoteView(render_window)
        
        server.start()


def main():
    if len(sys.argv) != 2 or sys.argv[1] not in ['vtk', 'trame']:
        print("Usage:")
        print("  python test_multibutton_comparison.py vtk    # Native VTK")
        print("  python test_multibutton_comparison.py trame  # Trame/web")
        print("")
        print("This demonstrates the difference in multi-button behavior:")
        print("  VTK:   Multi-button combinations work")
        print("  Trame: Multi-button combinations blocked by browser")
        sys.exit(1)
    
    mode = sys.argv[1]
    run_mode(mode)


if __name__ == "__main__":
    main()

Expected behavior

1. To see the expected behavior, run the pure VTK version of the same script by passing the "vtk" argument:

./test_multibutton_comparison.py vtk

2. In the VTK render window that opens, follow the same steps: press the left button, drag, press the right button, drag, release the left button, drag, release the right button, drag. You will get the expected output similar to the following:

LEFT BUTTON PRESS
DRAG: L=True R=False delta=[0,0]
DRAG: L=True R=False delta=[0,-1]
RIGHT BUTTON PRESS
DRAG: L=True R=True delta=[0,-1]
  *** LEFT+RIGHT COMBINATION DETECTED! ***
DRAG: L=True R=True delta=[0,-1]
  *** LEFT+RIGHT COMBINATION DETECTED! ***
LEFT BUTTON RELEASE
DRAG: L=False R=True delta=[0,0]
DRAG: L=False R=True delta=[0,-1]
RIGHT BUTTON RELEASE

Screenshots

N/A

Platform:

Device:

• Desktop
• Mobile => Not tested

OS:

• Windows => Not tested
• MacOS
• Linux
• Android => Not tested
• iOS => Not tested

Browsers Affected:

• Chrome
• Firefox
• Microsoft Edge => Not tested
• Safari
• Opera => Not tested
• Brave
• IE 11 => Not tested

[sudomakeinstall]

For me this is not necessarily a blocking issue, since I can work around it in my application by using other key combinations (left click plus shift, left click plus alt, etc). However, since I was able to isolate the difference between the pure VTK and VTK-in-Trame behavior, I wanted to document it here in case it is helpful to anyone (including myself in the future).

Of course, if this is not really a Trame bug but just user-error on my part, I'm very happy to be corrected! And, as always, thanks to everyone who has worked on this very useful and powerful library.

[jourdain]

It is most-likely a limitation of the networking implementation of mouse event in trame-vtk. Normally trame-rca should be more accurate at passing "all" VTK events over the network. Here is an example of how to use it.

[sudomakeinstall]

@jourdain I really appreciate the fast response! I was unaware of trame-rca, thank you for passing on the example. I added a test case using trame-rca and unfortunately the behavior is the same as without. Here is the complete example to reproduce:

#!/usr/bin/env python3
"""
Multi-button test comparison: VTK vs Trame vs Trame-RCA
Usage:
    ./test_multibutton_comparison.py vtk       # Native VTK (multi-button works)
    ./test_multibutton_comparison.py trame     # Trame/web (multi-button blocked)
    ./test_multibutton_comparison.py trame-rca # Trame with RCA (multi-button blocked)
"""

import sys
import vtk


class MultiButtonInteractor(vtk.vtkInteractorStyle):
    def __init__(self):
        super().__init__()
        self.left_button_down = False
        self.right_button_down = False
        self.last_mouse_pos = [0, 0]

        self.AddObserver("LeftButtonPressEvent", self.on_left_press)
        self.AddObserver("LeftButtonReleaseEvent", self.on_left_release)
        self.AddObserver("RightButtonPressEvent", self.on_right_press)
        self.AddObserver("RightButtonReleaseEvent", self.on_right_release)
        self.AddObserver("MouseMoveEvent", self.on_mouse_move)

    def on_left_press(self, obj, event):
        print("LEFT BUTTON PRESS")
        self.left_button_down = True
        interactor = self.GetInteractor()
        if interactor:
            self.last_mouse_pos = list(interactor.GetEventPosition())

    def on_left_release(self, obj, event):
        print("LEFT BUTTON RELEASE")
        self.left_button_down = False

    def on_right_press(self, obj, event):
        print("RIGHT BUTTON PRESS")
        self.right_button_down = True
        interactor = self.GetInteractor()
        if interactor:
            self.last_mouse_pos = list(interactor.GetEventPosition())

    def on_right_release(self, obj, event):
        print("RIGHT BUTTON RELEASE")
        self.right_button_down = False

    def on_mouse_move(self, obj, event):
        if self.left_button_down or self.right_button_down:
            interactor = self.GetInteractor()
            if interactor:
                current_pos = list(interactor.GetEventPosition())
                dx = current_pos[0] - self.last_mouse_pos[0]
                dy = current_pos[1] - self.last_mouse_pos[1]
                self.last_mouse_pos = current_pos

                print(
                    f"DRAG: L={self.left_button_down} R={self.right_button_down} delta=[{dx},{dy}]"
                )

                if self.left_button_down and self.right_button_down:
                    print("  *** LEFT+RIGHT COMBINATION DETECTED! ***")


def create_vtk_scene():
    """Shared VTK scene setup"""
    renderer = vtk.vtkRenderer()
    render_window = vtk.vtkRenderWindow()
    render_window.AddRenderer(renderer)

    interactor = vtk.vtkRenderWindowInteractor()
    interactor.SetRenderWindow(render_window)
    style = MultiButtonInteractor()
    interactor.SetInteractorStyle(style)

    return renderer, render_window, interactor


def run_mode(mode):
    if mode == "vtk":
        print("=== VTK MODE (Native) ===")
        print("Multi-button combinations SHOULD work")
        print("Try: Left-drag, then add Right button while dragging")
        print("")

        renderer, render_window, interactor = create_vtk_scene()
        render_window.SetWindowName("VTK Mode - Multi-button should work")

        interactor.Initialize()
        render_window.Render()
        interactor.Start()

    elif mode == "trame":
        print("=== TRAME MODE (Web) ===")
        print("Multi-button combinations should be BLOCKED")
        print("Try: Left-drag, then add Right button while dragging")
        print("Opening browser...")
        print("")

        from trame.app import get_server
        from trame.ui.vuetify3 import SinglePageLayout
        from trame.widgets import vtk as vtk_widgets

        server = get_server()

        renderer, render_window, interactor = create_vtk_scene()
        render_window.SetOffScreenRendering(True)

        with SinglePageLayout(server) as layout:
            layout.title.set_text("Trame Mode - Multi-button is blocked")
            with layout.content:
                vtk_widgets.VtkRemoteView(render_window)

        server.start()

    elif mode == "trame-rca":
        print("=== TRAME-RCA MODE (Web with RCA) ===")
        print("Multi-button combinations test with trame-rca")
        print("Try: Left-drag, then add Right button while dragging")
        print("Opening browser...")
        print("")

        from trame.app import get_server
        from trame.ui.vuetify3 import SinglePageLayout
        from trame_rca.widgets import rca

        server = get_server(client_type="vue3")

        renderer, render_window, interactor = create_vtk_scene()
        render_window.SetOffScreenRendering(True)

        with SinglePageLayout(server) as layout:
            layout.title.set_text("Trame-RCA Mode - Multi-button is blocked")
            with layout.content:
                view = rca.RemoteControlledArea(
                    name="view",
                    display="image",
                )
                view_handler = view.create_view_handler(render_window)

        server.start()


def main():
    if len(sys.argv) != 2 or sys.argv[1] not in ["vtk", "trame", "trame-rca"]:
        print("Usage:")
        print("  ./test_multibutton_comparison.py vtk       # Native VTK")
        print("  ./test_multibutton_comparison.py trame     # Trame/web")
        print("  ./test_multibutton_comparison.py trame-rca # Trame with RCA")
        print("")
        print("This demonstrates the difference in multi-button behavior:")
        print("  VTK:       Multi-button combinations work")
        print("  Trame:     Multi-button combinations blocked by browser")
        print("  Trame-RCA: Multi-button combinations test with RCA")
        sys.exit(1)

    mode = sys.argv[1]
    run_mode(mode)


if __name__ == "__main__":
    main()

But thank you nonetheless for alerting me to trame-rca--although it doesn't solve this specific problem for me, I think there are other ways I could use it to improve my application based on the example you shared.

[jourdain]

Thanks for providing those test cases/examples. We probably won't have the time to investigate them right aways as they are not critical, but it will definitely be great for when we tackle the issue!

[sudomakeinstall]

OK I'm going to abandon this for now, but before I do I just wanted to document one more data point in case it is helpful. By specifying the interactor_events and event_listeners arguments in the VtkRemoteView constructor, I can see that it's not that no event is triggered, but rather the wrong event is triggered. Specifically, you get a StartMouseMove and EndMouseMove. I didn't realize this initially because both are mapped to NoEvent and so don't trigger anything in the interactor. Here's a simplified example:

#!/usr/bin/env python3
"""
Multi-button test comparison: VTK vs Trame vs Trame-RCA
Usage:
    ./test_multibutton_comparison.py vtk       # Native VTK (multi-button works)
    ./test_multibutton_comparison.py trame     # Trame/web (multi-button blocked)
    ./test_multibutton_comparison.py trame-rca # Trame with RCA (multi-button blocked)
"""

import sys
import vtk

# Here are the events provided by vtkCommand:
# https://vtk.org/doc/nightly/html/classvtkCommand.html

# Here are the events provided by vtk-js:
# https://github.com/Kitware/vtk-js/blob/master/Sources/Rendering/Core/RenderWindowInteractor/index.js

# Here are the ones that are actually mapped in the VTK remote renderer (many mapped to vtkCommand:NoEvent):
# https://github.com/Kitware/VTK/blob/master/Web/Core/vtkRemoteInteractionAdapter.cxx

# Suspect problem is somewhere here:
# https://github.com/Kitware/VTK/blob/master/Web/Core/vtkWebApplication.cxx

VTK_VIEW_EVENTS = [
    "LeftButtonPress",
    "LeftButtonRelease",
    "RightButtonPress",
    "RightButtonRelease",
    "StartMouseMove",  # Mapped to NoEvent, never called in interactor
    "MouseMove",
    "EndMouseMove",  # Mapped to NoEvent, never called in interactor
]


def on_event(*args, **kwargs):
    print("event", args, kwargs, flush=True)


def event_listeners(events):
    result = {}
    for event in events:
        result[event] = (on_event, "[utils.vtk.event($event)]")
    return result


class MultiButtonInteractor(vtk.vtkInteractorStyle):
    def __init__(self):
        super().__init__()
        self.AddObserver("AnyEvent", self.print_event)  # This is never called

    def print_event(self, obj, event):
        print(event, flush=True)


def create_vtk_scene():
    """Shared VTK scene setup"""
    renderer = vtk.vtkRenderer()
    render_window = vtk.vtkRenderWindow()
    render_window.AddRenderer(renderer)

    interactor = vtk.vtkRenderWindowInteractor()
    interactor.SetRenderWindow(render_window)
    style = MultiButtonInteractor()
    interactor.SetInteractorStyle(style)

    return renderer, render_window, interactor


def run_mode(mode):
    if mode == "vtk":
        print("=== VTK MODE (Native) ===")
        print("Multi-button combinations SHOULD work")
        print("Try: Left-drag, then add Right button while dragging")
        print("")

        renderer, render_window, interactor = create_vtk_scene()
        render_window.SetWindowName("VTK Mode - Multi-button should work")

        interactor.Initialize()
        render_window.Render()
        interactor.Start()

    elif mode == "trame":
        print("=== TRAME MODE (Web) ===")
        print("Multi-button combinations should be BLOCKED")
        print("Try: Left-drag, then add Right button while dragging")
        print("Opening browser...")
        print("")

        from trame.app import get_server
        from trame.ui.vuetify3 import SinglePageLayout
        from trame.widgets import vtk as vtk_widgets

        server = get_server()

        renderer, render_window, interactor = create_vtk_scene()
        render_window.SetOffScreenRendering(True)

        with SinglePageLayout(server) as layout:
            layout.title.set_text("Trame Mode - Multi-button is blocked")
            with layout.content:
                vtk_widgets.VtkRemoteView(
                    render_window,
                    interactor_events=("event_types", VTK_VIEW_EVENTS),
                    **event_listeners(VTK_VIEW_EVENTS),
                )

        server.start(port=8050)

    elif mode == "trame-rca":
        print("=== TRAME-RCA MODE (Web with RCA) ===")
        print("Multi-button combinations test with trame-rca")
        print("Try: Left-drag, then add Right button while dragging")
        print("Opening browser...")
        print("")

        from trame.app import get_server
        from trame.ui.vuetify3 import SinglePageLayout
        from trame_rca.widgets import rca

        server = get_server(client_type="vue3")

        renderer, render_window, interactor = create_vtk_scene()
        render_window.SetOffScreenRendering(True)

        with SinglePageLayout(server) as layout:
            layout.title.set_text("Trame-RCA Mode - Multi-button is blocked")
            with layout.content:
                view = rca.RemoteControlledArea(
                    name="view",
                    display="image",
                )
                view_handler = view.create_view_handler(render_window)

        server.start(port=8050)


def main():
    if len(sys.argv) != 2 or sys.argv[1] not in ["vtk", "trame", "trame-rca"]:
        print("Usage:")
        print("  ./test_multibutton_comparison.py vtk       # Native VTK")
        print("  ./test_multibutton_comparison.py trame     # Trame/web")
        print("  ./test_multibutton_comparison.py trame-rca # Trame with RCA")
        print("")
        print("This demonstrates the difference in multi-button behavior:")
        print("  VTK:       Multi-button combinations work")
        print("  Trame:     Multi-button combinations blocked by browser")
        print("  Trame-RCA: Multi-button combinations test with RCA")
        sys.exit(1)

    mode = sys.argv[1]
    run_mode(mode)


if __name__ == "__main__":
    main()

Left Press:

MouseMoveEvent
LeftButtonPressEvent
event ({'type': 'LeftButtonPress', 'controlKey': False, 'altKey': False, 'shiftKey': False, 'position': {'x': 330.7531692766592, 'y': 446.70381150579914, 'z': 0}},) {}

Left Release:

MouseMoveEvent
LeftButtonReleaseEvent
event ({'type': 'LeftButtonRelease', 'controlKey': False, 'altKey': False, 'shiftKey': False, 'position': {'x': 330.7531692766592, 'y': 446.70381150579914, 'z': 0}},) {}

Right Press:

MouseMoveEvent
RightButtonPressEvent
event ({'type': 'RightButtonPress', 'controlKey': False, 'altKey': False, 'shiftKey': False, 'position': {'x': 330.7531692766592, 'y': 446.70381150579914, 'z': 0}},) {}

Right Release:

MouseMoveEvent
RightButtonReleaseEvent
event ({'type': 'RightButtonRelease', 'controlKey': False, 'altKey': False, 'shiftKey': False, 'position': {'x': 330.7531692766592, 'y': 446.70381150579914, 'z': 0}},) {}

Left Press:

MouseMoveEvent
LeftButtonPressEvent
event ({'type': 'LeftButtonPress', 'controlKey': False, 'altKey': False, 'shiftKey': False, 'position': {'x': 330.7531692766592, 'y': 446.70381150579914, 'z': 0}},) {}

Right Press:

event ({'type': 'StartMouseMove', 'controlKey': False, 'altKey': False, 'shiftKey': False, 'position': {'x': 330.7531692766592, 'y': 446.70381150579914, 'z': 0}},) {}
event ({'type': 'EndMouseMove'},) {}

I tracked this back as far as wslink/protocol.py:256, the call to onMessage and see that StartMouseMove and EndMouseMove are indeed what's being passed, so I think the issue must be higher up in the chain. Beyond that the data is encoded and I don't know how to interpret it.

Anyway, sorry I wasn't able to track it down, but hope this helps someone at some point.

[jourdain]

FYI, this is where the event handling is getting processed for trame-vtk.

https://github.com/Kitware/trame-vtk/blob/master/src/trame_vtk/modules/vtk/protocols/mouse_handler.py