From ce2936bd00220578a0fdc482b68bfed81b86e9ac Mon Sep 17 00:00:00 2001 From: yuecideng Date: Wed, 21 Jan 2026 20:04:23 +0800 Subject: [PATCH 1/2] wip --- docs/source/features/interaction/index.rst | 10 ++++++ docs/source/features/interaction/window.md | 39 ++++++++++++++++++++ docs/source/index.rst | 1 + embodichain/lab/sim/sim_manager.py | 42 ++++++++++++++++++++-- 4 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 docs/source/features/interaction/index.rst create mode 100644 docs/source/features/interaction/window.md diff --git a/docs/source/features/interaction/index.rst b/docs/source/features/interaction/index.rst new file mode 100644 index 00000000..7640c1f1 --- /dev/null +++ b/docs/source/features/interaction/index.rst @@ -0,0 +1,10 @@ +Interactive Simulation +====================== + +The Interactive Simulation module provides tools and interfaces for interacting with the simulation environment, including window management, input handling, and real-time control of simulated assets. + +.. toctree:: + :maxdepth: 2 + + Window interaction + \ No newline at end of file diff --git a/docs/source/features/interaction/window.md b/docs/source/features/interaction/window.md new file mode 100644 index 00000000..a037e1b3 --- /dev/null +++ b/docs/source/features/interaction/window.md @@ -0,0 +1,39 @@ +# Window interaction + +This section describes the default window interaction controls available in the simulation. These controls allow users to interact with the simulation environment using keyboard, mouse, and customizable input events. + +## Default Window Events + +The simulation window comes with a set of default controls that enable users to perform various actions, such as selecting objects, manipulating the camera view, and triggering specific events. These controls are implemented using the `ObjectManipulator` class (provided by `dexsim`). + +| Events | Description | +|---------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------| +| **Raycast Information Display** | Press the right mouse button and the 'C' key to print the raycast distance and hit position of a surface (world coordinates) to the console. Useful for debugging and checking the position of objects in the simulation. | + +> **Note:** We will add more interaction features in future releases. Stay tuned for updates! + +## Customizing Window Events + +Users can create their own custom window interaction controls by subclassing the `ObjectManipulator` class. This allows for the implementation of specific behaviors and responses to user inputs. + +Here's an example of how to create a custom window event that responds to key presses: + +```python +from dexsim.engine import ObjectManipulator +from dexsim.types import InputKey + +class CustomWindowEvent(ObjectManipulator): + def on_key_down(self, key): + if key == InputKey.SPACE: + print("Space key pressed!") +``` + +The functions table below summarizes the key methods available in the `ObjectManipulator` class for customizing window events: +| Method | Description | +|----------------------|---------------------------------------------------------------------------------------------------| +| `on_key_down(key)` | Triggered when a key is pressed down. The `key` parameter indicates which key was pressed. | +| `on_key_up(key)` | Triggered when a key is released. The `key` parameter indicates which key was released. | +| `on_mouse_moved(x, y)`| Triggered when the mouse is moved. The `x` and `y` parameters indicate the new mouse position. | +| `on_mouse_down(button, x, y)` | Triggered when a mouse button is pressed. The `button` parameter indicates which button was pressed, and `x`, `y` indicate the mouse position. | +| `on_mouse_up(button, x, y)` | Triggered when a mouse button is released. The `button` parameter indicates which button was released, and `x`, `y` indicate the mouse position. | +| `on_mouse_wheel(delta)` | Triggered when the mouse wheel is scrolled. The `delta` parameter indicates the amount of scroll. | diff --git a/docs/source/index.rst b/docs/source/index.rst index c0ac442e..c3ef7343 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -37,6 +37,7 @@ Table of Contents :glob: features/workspace_analyzer/index* + features/interaction/index* .. toctree:: :maxdepth: 1 diff --git a/embodichain/lab/sim/sim_manager.py b/embodichain/lab/sim/sim_manager.py index c1aec881..d47d7b21 100644 --- a/embodichain/lab/sim/sim_manager.py +++ b/embodichain/lab/sim/sim_manager.py @@ -47,8 +47,8 @@ ) from dexsim.engine import CudaArray, Material from dexsim.models import MeshObject -from dexsim.render import Light as _Light, LightType -from dexsim.engine import GizmoController +from dexsim.render import Light as _Light, LightType, Windows +from dexsim.engine import GizmoController, ObjectManipulator from embodichain.lab.sim.objects import ( RigidObject, @@ -214,7 +214,13 @@ def __init__( # Initialize warp runtime context before creating the world. wp.init() - self._world = dexsim.World(world_config) + self._world: dexsim.World = dexsim.World(world_config) + + self._window: Windows = None + self._is_registered_window_control = False + if sim_config.headless is False: + self._window = self._world.get_windows() + self._register_default_window_control() fps = int(1.0 / sim_config.physics_dt) self._world.set_physics_fps(fps) @@ -574,6 +580,8 @@ def get_world(self) -> dexsim.World: def open_window(self) -> None: """Open the simulation window.""" self._world.open_window() + self._window = self._world.get_windows() + self._register_default_window_control() self.is_window_opened = True def close_window(self) -> None: @@ -1502,6 +1510,34 @@ def remove_marker(self, name: str) -> bool: logger.log_warning(f"Failed to remove marker {name}: {str(e)}") return False + def _register_default_window_control(self) -> None: + """Register default window controls for better simulation interaction.""" + from dexsim.types import InputKey + + if self._is_registered_window_control: + return + + class WindowDefaultEvent(ObjectManipulator): + + def on_key_down(self, key): + if key == InputKey.SCANCODE_C.value: + print(f"Raycast distance: {self.selected_distance}") + print(f"Hit position: {self.selected_position}") + + manipulator = WindowDefaultEvent() + manipulator.enable_selection_cache(True) + self._window.add_input_control(manipulator) + + self._is_registered_window_control = True + + def add_custom_window_control(self, controls: list[ObjectManipulator]) -> None: + if self._window is None: + logger.log_warning("No window available to add custom controls.") + return + + for control in controls: + self._window.add_input_control(control) + def create_visual_material(self, cfg: VisualMaterialCfg) -> VisualMaterial: """Create a visual material with given configuration. From e95e518b12d924eeac926e7a50971c7fde555905 Mon Sep 17 00:00:00 2001 From: yuecideng Date: Thu, 22 Jan 2026 00:40:32 +0800 Subject: [PATCH 2/2] wip --- docs/source/features/interaction/window.md | 12 ++++++++++-- embodichain/lab/sim/sim_manager.py | 15 ++++++++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/source/features/interaction/window.md b/docs/source/features/interaction/window.md index a037e1b3..6c512186 100644 --- a/docs/source/features/interaction/window.md +++ b/docs/source/features/interaction/window.md @@ -8,7 +8,7 @@ The simulation window comes with a set of default controls that enable users to | Events | Description | |---------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------| -| **Raycast Information Display** | Press the right mouse button and the 'C' key to print the raycast distance and hit position of a surface (world coordinates) to the console. Useful for debugging and checking the position of objects in the simulation. | +| **Raycast Information Display** | Press the right mouse button to select a point and the 'C' key to print the raycast distance and hit position of a surface (world coordinates) to the console. Useful for debugging and checking the position of objects in the simulation. | > **Note:** We will add more interaction features in future releases. Stay tuned for updates! @@ -24,8 +24,16 @@ from dexsim.types import InputKey class CustomWindowEvent(ObjectManipulator): def on_key_down(self, key): - if key == InputKey.SPACE: + if key == InputKey.SPACE.value: print("Space key pressed!") + + +# Assuming you already have a SimulationManager instance called `sim_manager` +# (for example, created elsewhere in your code): +# sim_manager = SimulationManager(...) + +# Register the custom window event handler with the simulation: +sim_manager.add_custom_window_control(CustomWindowEvent()) ``` The functions table below summarizes the key methods available in the `ObjectManipulator` class for customizing window events: diff --git a/embodichain/lab/sim/sim_manager.py b/embodichain/lab/sim/sim_manager.py index d47d7b21..c2f078b0 100644 --- a/embodichain/lab/sim/sim_manager.py +++ b/embodichain/lab/sim/sim_manager.py @@ -216,7 +216,7 @@ def __init__( wp.init() self._world: dexsim.World = dexsim.World(world_config) - self._window: Windows = None + self._window: Windows | None = None self._is_registered_window_control = False if sim_config.headless is False: self._window = self._world.get_windows() @@ -1531,6 +1531,19 @@ def on_key_down(self, key): self._is_registered_window_control = True def add_custom_window_control(self, controls: list[ObjectManipulator]) -> None: + """Add one or more custom window input controls. + + This method registers additional :class:`ObjectManipulator` instances + with the simulation window so they can handle input events alongside + any default controls. + + Args: + controls (list[ObjectManipulator]): A list of initialized + ObjectManipulator instances to add to the current window. + Each control will be registered via ``window.add_input_control``. + If no window is available, the controls are not added and a + warning is logged. + """ if self._window is None: logger.log_warning("No window available to add custom controls.") return