From 98215aed4caa2beaf4ff157a6ca24df0b9409ab2 Mon Sep 17 00:00:00 2001 From: Faisal Zaghloul Date: Mon, 18 Aug 2025 19:28:54 -0400 Subject: [PATCH 1/8] Initial version of spot observer client --- Assets/CVDDataGenerator.cs | 20 +- Assets/DepthCompletion.cs | 26 +- Assets/DepthManager.cs | 38 +- Assets/ICPLauncher.cs | 6 +- Assets/OpticalFlowEstimator.cs | 26 +- Assets/Scenes/ghost.unity | 412 +++++++++--------- Assets/Scripts/DrawMeshInstanced.cs | 31 +- Assets/Scripts/SpotObserverClient.cs | 228 ++++++++++ Assets/Scripts/SpotObserverClient.cs.meta | 2 + Assets/Scripts/UBModel.cs | 8 +- .../spot_description/meshes/body.dae.meta | 5 +- .../meshes/front_left_hip.dae.meta | 5 +- .../meshes/front_left_lower_leg.dae.meta | 5 +- .../meshes/front_left_upper_leg.dae.meta | 5 +- .../meshes/front_right_hip.dae.meta | 5 +- .../meshes/front_right_lower_leg.dae.meta | 5 +- .../meshes/front_right_upper_leg.dae.meta | 5 +- .../meshes/rear_left_hip.dae.meta | 5 +- .../meshes/rear_left_lower_leg.dae.meta | 5 +- .../meshes/rear_left_upper_leg.dae.meta | 5 +- .../meshes/rear_right_hip.dae.meta | 5 +- .../meshes/rear_right_lower_leg.dae.meta | 5 +- .../meshes/rear_right_upper_leg.dae.meta | 5 +- .../spot/meshes/arm0_link_el0.dae.meta | 5 +- .../spot/meshes/arm0_link_el1.dae.meta | 5 +- .../spot/meshes/arm0_link_fngr.dae.meta | 5 +- .../spot/meshes/arm0_link_hr0.dae.meta | 5 +- .../spot/meshes/arm0_link_sh0.dae.meta | 5 +- .../spot/meshes/arm0_link_sh1.dae.meta | 5 +- .../spot/meshes/arm0_link_wr0.dae.meta | 5 +- .../spot/meshes/arm0_link_wr1.dae.meta | 5 +- .../XR/Settings/OpenXR Package Settings.asset | 136 +++++- .../onnx/2camera_top20_45Smallmodel.onnx.meta | 5 +- Assets/onnx/baselineMask.onnx.meta | 5 +- Assets/onnx/neuflow_mixed.onnx.meta | 3 +- Assets/onnx/train_multi_step2.onnx.meta | 5 +- Assets/onnx/yolov8n.onnx.meta | 3 +- Packages/manifest.json | 6 +- Packages/packages-lock.json | 62 ++- ProjectSettings/ProjectSettings.asset | 13 +- ProjectSettings/ProjectVersion.txt | 4 +- 41 files changed, 805 insertions(+), 339 deletions(-) create mode 100644 Assets/Scripts/SpotObserverClient.cs create mode 100644 Assets/Scripts/SpotObserverClient.cs.meta diff --git a/Assets/CVDDataGenerator.cs b/Assets/CVDDataGenerator.cs index bc88442..0c5b606 100644 --- a/Assets/CVDDataGenerator.cs +++ b/Assets/CVDDataGenerator.cs @@ -1,15 +1,15 @@ using RosSharp.RosBridgeClient; using System.Collections; using System.Collections.Generic; -using Unity.Sentis; + using UnityEngine; public class CVDDataGenerator : MonoBehaviour { private bool first_run = true; - Tensor previous_rgbL; - Tensor previous_rgbR; + Unity.InferenceEngine.Tensor previous_rgbL; + Unity.InferenceEngine.Tensor previous_rgbR; //public BodyPoseSubscriber pose_subscriber; @@ -133,20 +133,20 @@ IEnumerator SetFirstRunFalseAfterDelay(float delay) //} - public (ComputeBuffer, ComputeBuffer, Matrix4x4, Matrix4x4, ComputeBuffer, ComputeBuffer) generatePoseData(Tensor depthL, Tensor rgbL, Tensor depthR, Tensor rgbR, bool activate_depth_completion, bool activate_CVD) + public (ComputeBuffer, ComputeBuffer, Matrix4x4, Matrix4x4, ComputeBuffer, ComputeBuffer) generatePoseData(Unity.InferenceEngine.Tensor depthL, Unity.InferenceEngine.Tensor rgbL, Unity.InferenceEngine.Tensor depthR, Unity.InferenceEngine.Tensor rgbR, bool activate_depth_completion, bool activate_CVD) { if (first_run) { - previous_rgbL = new Tensor(rgbL.shape); - previous_rgbR = new Tensor(rgbR.shape); + previous_rgbL = new Unity.InferenceEngine.Tensor(rgbL.shape); + previous_rgbR = new Unity.InferenceEngine.Tensor(rgbR.shape); //buffer_depthL?.Release(); //buffer_depthR?.Release(); //buffer_opticalL?.Release(); //buffer_opticalR?.Release(); - buffer_depthL = ComputeTensorData.Pin(depthL).buffer; - buffer_depthR = ComputeTensorData.Pin(depthR).buffer; + buffer_depthL = Unity.InferenceEngine.ComputeTensorData.Pin(depthL).buffer; + buffer_depthR = Unity.InferenceEngine.ComputeTensorData.Pin(depthR).buffer; buffer_opticalL = new ComputeBuffer(480 * 640 * 2, sizeof(float)); buffer_opticalR = new ComputeBuffer(480 * 640 * 2, sizeof(float)); @@ -167,8 +167,8 @@ IEnumerator SetFirstRunFalseAfterDelay(float delay) { //buffer_depthL?.Release(); //buffer_depthR?.Release(); - buffer_depthL = ComputeTensorData.Pin(depthL).buffer; - buffer_depthR = ComputeTensorData.Pin(depthR).buffer; + buffer_depthL = Unity.InferenceEngine.ComputeTensorData.Pin(depthL).buffer; + buffer_depthR = Unity.InferenceEngine.ComputeTensorData.Pin(depthR).buffer; } diff --git a/Assets/DepthCompletion.cs b/Assets/DepthCompletion.cs index fe41170..c04040d 100644 --- a/Assets/DepthCompletion.cs +++ b/Assets/DepthCompletion.cs @@ -2,24 +2,24 @@ using System.Collections; using System.Collections.Generic; using System.Runtime.InteropServices; -using Unity.Sentis; + using UnityEditor; using UnityEngine; using UnityEngine.VFX; -using static Unity.Sentis.Model; +using static Unity.InferenceEngine.Model; public class DepthCompletion : MonoBehaviour { - public ModelAsset rgbdModelAsset; - Model runtimeModelRGBD; - Worker workerRGBD; + public Unity.InferenceEngine.ModelAsset rgbdModelAsset; + Unity.InferenceEngine.Model runtimeModelRGBD; + Unity.InferenceEngine.Worker workerRGBD; public bool Use_UB; // Use UnityBYOM for inference public UBModel UB_Model; // Reference to UBModel script private int run_nconv; - Tensor depth_outputTensor_0, depth_outputTensor_1; + Unity.InferenceEngine.Tensor depth_outputTensor_0, depth_outputTensor_1; ComputeBuffer computeTensorData0, computeTensorData1; @@ -29,8 +29,8 @@ public class DepthCompletion : MonoBehaviour void Start() { // Setup sentis model - runtimeModelRGBD = ModelLoader.Load(rgbdModelAsset); - workerRGBD = new Worker(runtimeModelRGBD, BackendType.GPUCompute); + runtimeModelRGBD = Unity.InferenceEngine.ModelLoader.Load(rgbdModelAsset); + workerRGBD = new Unity.InferenceEngine.Worker(runtimeModelRGBD, Unity.InferenceEngine.BackendType.GPUCompute); run_nconv = 2; } @@ -47,7 +47,7 @@ void OnDestroy() // =============================================================================== // // Depth Completion // // =============================================================================== // - public (ComputeBuffer, ComputeBuffer) complete(Tensor depth_tensor_0, Tensor color_tensor_0, Tensor depth_tensor_1, Tensor color_tensor_1) + public (ComputeBuffer, ComputeBuffer) complete(Unity.InferenceEngine.Tensor depth_tensor_0, Unity.InferenceEngine.Tensor color_tensor_0, Unity.InferenceEngine.Tensor depth_tensor_1, Unity.InferenceEngine.Tensor color_tensor_1) { if (Use_UB) { @@ -88,11 +88,11 @@ void OnDestroy() workerRGBD.Schedule(); - Tensor depth_outputTensor_0 = workerRGBD.PeekOutput("output_depth_0") as Tensor; - ComputeBuffer computeTensorData0 = ComputeTensorData.Pin(depth_outputTensor_0).buffer; + Unity.InferenceEngine.Tensor depth_outputTensor_0 = workerRGBD.PeekOutput("output_depth_0") as Unity.InferenceEngine.Tensor; + ComputeBuffer computeTensorData0 = Unity.InferenceEngine.ComputeTensorData.Pin(depth_outputTensor_0).buffer; - Tensor depth_outputTensor_1 = workerRGBD.PeekOutput("output_depth_1") as Tensor; - ComputeBuffer computeTensorData1 = ComputeTensorData.Pin(depth_outputTensor_1).buffer; + Unity.InferenceEngine.Tensor depth_outputTensor_1 = workerRGBD.PeekOutput("output_depth_1") as Unity.InferenceEngine.Tensor; + ComputeBuffer computeTensorData1 = Unity.InferenceEngine.ComputeTensorData.Pin(depth_outputTensor_1).buffer; return (computeTensorData0, computeTensorData1); } diff --git a/Assets/DepthManager.cs b/Assets/DepthManager.cs index d8e7654..8205d0d 100644 --- a/Assets/DepthManager.cs +++ b/Assets/DepthManager.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using UnityEngine; using UnityEngine.UI; -using Unity.Sentis; + using Unity.VisualScripting; public class DepthManager : MonoBehaviour @@ -29,11 +29,11 @@ public class DepthManager : MonoBehaviour public DepthAveraging AveragerLeft, AveragerRight; - private Tensor depth_left_t_1; - private Tensor rgb_left_t_1; + private Unity.InferenceEngine.Tensor depth_left_t_1; + private Unity.InferenceEngine.Tensor rgb_left_t_1; - private Tensor depth_right_t_1; - private Tensor rgb_right_t_1; + private Unity.InferenceEngine.Tensor depth_right_t_1; + private Unity.InferenceEngine.Tensor rgb_right_t_1; private bool received_left_1 = false; private bool received_right_1 = false; @@ -74,8 +74,8 @@ public class DepthManager : MonoBehaviour private ComputeBuffer final_out; - TensorShape depth_shape = new TensorShape(1, 1, 480, 640); - TensorShape color_shape = new TensorShape(1, 3, 480, 640); + Unity.InferenceEngine.TensorShape depth_shape = new Unity.InferenceEngine.TensorShape(1, 1, 480, 640); + Unity.InferenceEngine.TensorShape color_shape = new Unity.InferenceEngine.TensorShape(1, 3, 480, 640); Matrix4x4 ICP_trans = Matrix4x4.identity; Matrix4x4 icp_trans_temp = Matrix4x4.identity; @@ -125,11 +125,11 @@ void Start() //left_eye_data_timer_id = fps_timer.registerTimer("Left eye data"); //right_eye_data_timer_id = fps_timer.registerTimer("Right eye data"); - depth_left_t_1 = new Tensor(depth_shape); - depth_right_t_1 = new Tensor(depth_shape); + depth_left_t_1 = new Unity.InferenceEngine.Tensor(depth_shape); + depth_right_t_1 = new Unity.InferenceEngine.Tensor(depth_shape); - rgb_left_t_1 = new Tensor(color_shape, data: null); - rgb_right_t_1 = new Tensor(color_shape, data: null); + rgb_left_t_1 = new Unity.InferenceEngine.Tensor(color_shape, data: null); + rgb_right_t_1 = new Unity.InferenceEngine.Tensor(color_shape, data: null); received_left_1 = false; received_right_1 = false; @@ -209,7 +209,7 @@ private IEnumerator ResetActivateDepthEstimation() public (ComputeBuffer, Matrix4x4, float[]) update_depth_from_renderer(Texture2D rgb, float[] depth, int camera_index, bool calculate_icp, bool new_depth, bool avg_before_complete) { - TextureTransform tform = new(); + Unity.InferenceEngine.TextureTransform tform = new(); tform.SetDimensions(rgb.width, rgb.height, 3); @@ -237,10 +237,10 @@ private IEnumerator ResetActivateDepthEstimation() } else { - depth_left_t_1 = new Tensor(depth_shape, left_depth_avg); + depth_left_t_1 = new Unity.InferenceEngine.Tensor(depth_shape, left_depth_avg); //depth_left_t_1 = new Tensor(depth_shape, depth); } - TextureConverter.ToTensor(rgb, rgb_left_t_1, tform); + Unity.InferenceEngine.TextureConverter.ToTensor(rgb, rgb_left_t_1, tform); rgb_left_t_1.Reshape(color_shape); received_left_1 = true; @@ -259,10 +259,10 @@ private IEnumerator ResetActivateDepthEstimation() } else { - depth_right_t_1 = new Tensor(depth_shape, right_depth_avg); + depth_right_t_1 = new Unity.InferenceEngine.Tensor(depth_shape, right_depth_avg); //depth_right_t_1 = new Tensor(depth_shape, depth); } - TextureConverter.ToTensor(rgb, rgb_right_t_1, tform); + Unity.InferenceEngine.TextureConverter.ToTensor(rgb, rgb_right_t_1, tform); rgb_right_t_1.Reshape(color_shape); received_right_1 = true; @@ -282,7 +282,7 @@ private IEnumerator ResetActivateDepthEstimation() return (temp_output_left_1, icp_trans_temp, right_depth_avg); } - private (ComputeBuffer, ComputeBuffer, Matrix4x4) process_depth(Tensor depthL_1, Tensor rgbL_1, Tensor depthR_1, Tensor rgbR_1, bool is_not_moving, bool calculate_icp) + private (ComputeBuffer, ComputeBuffer, Matrix4x4) process_depth(Unity.InferenceEngine.Tensor depthL_1, Unity.InferenceEngine.Tensor rgbL_1, Unity.InferenceEngine.Tensor depthR_1, Unity.InferenceEngine.Tensor rgbR_1, bool is_not_moving, bool calculate_icp) { ICP_trans = Matrix4x4.identity; if (calculate_icp && activate_ICP) @@ -294,8 +294,8 @@ private IEnumerator ResetActivateDepthEstimation() // Get compute buffers for depth - ComputeBuffer depthL_1_buf = ComputeTensorData.Pin(depthL_1).buffer; - ComputeBuffer depthR_1_buf = ComputeTensorData.Pin(depthR_1).buffer; + ComputeBuffer depthL_1_buf = Unity.InferenceEngine.ComputeTensorData.Pin(depthL_1).buffer; + ComputeBuffer depthR_1_buf = Unity.InferenceEngine.ComputeTensorData.Pin(depthR_1).buffer; AveragerLeft.prev_filling(depthL_1_buf); AveragerRight.prev_filling(depthR_1_buf); diff --git a/Assets/ICPLauncher.cs b/Assets/ICPLauncher.cs index 1ae0556..7ba399b 100644 --- a/Assets/ICPLauncher.cs +++ b/Assets/ICPLauncher.cs @@ -7,7 +7,7 @@ using MathNet.Numerics.LinearAlgebra; using MathNet.Numerics.LinearAlgebra.Single; using Meta.WitAi; -using Unity.Sentis; + using System.IO; using System; @@ -56,7 +56,7 @@ public class ICPLauncher : MonoBehaviour float3x3 R_all; float3 t_all; - TensorShape depth_shape; + Unity.InferenceEngine.TensorShape depth_shape; float global_min_error = 10000000000.0f; @@ -119,7 +119,7 @@ void Start() { depth3d_return = new float3[W * H]; - depth_shape = new TensorShape(1, 1, 480, 640); + depth_shape = new Unity.InferenceEngine.TensorShape(1, 1, 480, 640); buffer0 = new ComputeBuffer(480 * 640, sizeof(float)); buffer1 = new ComputeBuffer(480 * 640, sizeof(float)); buffer2 = new ComputeBuffer(480 * 640, sizeof(float)); diff --git a/Assets/OpticalFlowEstimator.cs b/Assets/OpticalFlowEstimator.cs index e47b216..8a24375 100644 --- a/Assets/OpticalFlowEstimator.cs +++ b/Assets/OpticalFlowEstimator.cs @@ -1,14 +1,14 @@ using System.Collections; using System.Collections.Generic; -using Unity.Sentis; + using UnityEngine; public class OpticalFlowEstimator : MonoBehaviour { - public ModelAsset NeuV2; - Model runtimeModelNeuV2; - Worker workerNeuV2R; - Worker workerNeuV2L; + public Unity.InferenceEngine.ModelAsset NeuV2; + Unity.InferenceEngine.Model runtimeModelNeuV2; + Unity.InferenceEngine.Worker workerNeuV2R; + Unity.InferenceEngine.Worker workerNeuV2L; //ComputeBuffer computeTensorData0; //ComputeBuffer computeTensorData1; @@ -16,10 +16,10 @@ public class OpticalFlowEstimator : MonoBehaviour // Start is called before the first frame update void Start() { - runtimeModelNeuV2 = ModelLoader.Load(NeuV2); - workerNeuV2R = new Worker(runtimeModelNeuV2, BackendType.GPUCompute); + runtimeModelNeuV2 = Unity.InferenceEngine.ModelLoader.Load(NeuV2); + workerNeuV2R = new Unity.InferenceEngine.Worker(runtimeModelNeuV2, Unity.InferenceEngine.BackendType.GPUCompute); - workerNeuV2L = new Worker(runtimeModelNeuV2, BackendType.GPUCompute); + workerNeuV2L = new Unity.InferenceEngine.Worker(runtimeModelNeuV2, Unity.InferenceEngine.BackendType.GPUCompute); } void OnDestroy() @@ -38,7 +38,7 @@ void Update() } - public (ComputeBuffer, ComputeBuffer) estimate_all(Tensor rgbL0, Tensor rgbL1, Tensor rgbR0, Tensor rgbR1) + public (ComputeBuffer, ComputeBuffer) estimate_all(Unity.InferenceEngine.Tensor rgbL0, Unity.InferenceEngine.Tensor rgbL1, Unity.InferenceEngine.Tensor rgbR0, Unity.InferenceEngine.Tensor rgbR1) { //Debug.Log("estimate Flow"); // left @@ -46,9 +46,9 @@ void Update() workerNeuV2L.SetInput("input2", rgbL1); workerNeuV2L.Schedule(); - Tensor depth_outputTensor_0 = workerNeuV2L.PeekOutput() as Tensor; + Unity.InferenceEngine.Tensor depth_outputTensor_0 = workerNeuV2L.PeekOutput() as Unity.InferenceEngine.Tensor; //depth_outputTensor_0.Reshape(new TensorShape(2, 480, 640)); - ComputeBuffer computeData0 = ComputeTensorData.Pin(depth_outputTensor_0).buffer; + ComputeBuffer computeData0 = Unity.InferenceEngine.ComputeTensorData.Pin(depth_outputTensor_0).buffer; //Debug.Log(depth_outputTensor_0); // right @@ -56,9 +56,9 @@ void Update() workerNeuV2R.SetInput("input2", rgbR1); workerNeuV2R.Schedule(); - Tensor depth_outputTensor_1 = workerNeuV2R.PeekOutput() as Tensor; + Unity.InferenceEngine.Tensor depth_outputTensor_1 = workerNeuV2R.PeekOutput() as Unity.InferenceEngine.Tensor; //depth_outputTensor_1.Reshape(new TensorShape(2, 480, 640)); - ComputeBuffer computeData1 = ComputeTensorData.Pin(depth_outputTensor_1).buffer; + ComputeBuffer computeData1 = Unity.InferenceEngine.ComputeTensorData.Pin(depth_outputTensor_1).buffer; //Debug.Log(depth_outputTensor_1); //depth_outputTensor_0.Dispose(); diff --git a/Assets/Scenes/ghost.unity b/Assets/Scenes/ghost.unity index b1f60c6..8840451 100644 --- a/Assets/Scenes/ghost.unity +++ b/Assets/Scenes/ghost.unity @@ -411,11 +411,6 @@ Transform: m_Children: [] m_Father: {fileID: 1499995371} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} ---- !u!4 &43937878 stripped -Transform: - m_CorrespondingSourceObject: {fileID: 1137478774281308966, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} - m_PrefabInstance: {fileID: 409472599} - m_PrefabAsset: {fileID: 0} --- !u!23 &43937879 stripped MeshRenderer: m_CorrespondingSourceObject: {fileID: 1137478774641359479, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} @@ -634,6 +629,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -655,6 +652,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -1146,74 +1144,6 @@ Transform: - {fileID: 631669220} m_Father: {fileID: 680693538} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} ---- !u!1 &128476700 -GameObject: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - serializedVersion: 6 - m_Component: - - component: {fileID: 128476701} - - component: {fileID: 128476703} - - component: {fileID: 128476702} - m_Layer: 0 - m_Name: HandDepthSubscriber - m_TagString: Untagged - m_Icon: {fileID: 0} - m_NavMeshLayer: 0 - m_StaticEditorFlags: 0 - m_IsActive: 1 ---- !u!4 &128476701 -Transform: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 128476700} - serializedVersion: 2 - m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} - m_LocalPosition: {x: 0, y: 0, z: 0} - m_LocalScale: {x: 1, y: 1, z: 1} - m_ConstrainProportionsScale: 0 - m_Children: [] - m_Father: {fileID: 43937878} - m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} ---- !u!114 &128476702 -MonoBehaviour: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 128476700} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: ff98778ae051cfd40a6e4effdf851ebd, type: 3} - m_Name: - m_EditorClassIdentifier: - SecondsTimeout: 10 - Serializer: 0 - protocol: 1 - RosBridgeServerUrl: ws://128.148.138.132:9090 ---- !u!114 &128476703 -MonoBehaviour: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 128476700} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: f88981aeb3483944785bdbed96c04bf4, type: 3} - m_Name: - m_EditorClassIdentifier: - Topic: /spot/stream_image/hand_depth_in_hand_color_frame/image - TimeStep: 0 - timestamp_synced: 0 - printMessageReceiveRate: 0 - printMessageProcRate: 0 - compressed: 0 - new_depth: 0 --- !u!1 &129872616 GameObject: m_ObjectHideFlags: 0 @@ -2392,6 +2322,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -2413,6 +2345,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -2435,17 +2368,6 @@ SkinnedMeshRenderer: m_Center: {x: 0.009499997, y: 0.01770607, z: -0.0011095554} m_Extent: {x: 0.04527676, y: 0.05372102, z: 0.06648618} m_DirtyAABB: 0 ---- !u!114 &260625234 stripped -MonoBehaviour: - m_CorrespondingSourceObject: {fileID: 1137478774821249055, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} - m_PrefabInstance: {fileID: 409472599} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 0} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: 441fcddae936d184f8196aeed41a9f18, type: 3} - m_Name: - m_EditorClassIdentifier: --- !u!1 &264593255 GameObject: m_ObjectHideFlags: 0 @@ -2681,6 +2603,8 @@ MeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -2702,6 +2626,7 @@ MeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -2766,6 +2691,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -2787,6 +2714,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -3084,6 +3012,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -3105,6 +3035,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -3173,6 +3104,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -3194,6 +3127,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -3415,6 +3349,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -3436,6 +3372,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -3581,6 +3518,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -3602,6 +3541,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -3800,17 +3740,6 @@ Transform: m_Children: [] m_Father: {fileID: 121168760} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} ---- !u!114 &383063648 stripped -MonoBehaviour: - m_CorrespondingSourceObject: {fileID: 1137478775531494914, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} - m_PrefabInstance: {fileID: 409472599} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 0} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: 441fcddae936d184f8196aeed41a9f18, type: 3} - m_Name: - m_EditorClassIdentifier: --- !u!1 &400188806 GameObject: m_ObjectHideFlags: 0 @@ -3895,6 +3824,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -3916,6 +3847,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -4243,11 +4175,11 @@ PrefabInstance: - target: {fileID: 1137478774599216621, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: colorSubscriber value: - objectReference: {fileID: 1953341855} + objectReference: {fileID: 0} - target: {fileID: 1137478774599216621, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: depthSubscriber value: - objectReference: {fileID: 128476703} + objectReference: {fileID: 0} - target: {fileID: 1137478774821249048, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: RosBridgeServerUrl value: ws://128.148.138.132:9090 @@ -4308,6 +4240,10 @@ PrefabInstance: propertyPath: use_saved_meshes value: 1 objectReference: {fileID: 0} + - target: {fileID: 1137478775010377568, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: spotObserverClient + value: + objectReference: {fileID: 971801071} - target: {fileID: 1137478775010377574, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: m_IsActive value: 1 @@ -4391,11 +4327,11 @@ PrefabInstance: - target: {fileID: 1137478775515482002, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: colorSubscriber value: - objectReference: {fileID: 383063648} + objectReference: {fileID: 0} - target: {fileID: 1137478775515482002, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: depthSubscriber value: - objectReference: {fileID: 941303697} + objectReference: {fileID: 0} - target: {fileID: 1137478775515482002, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: imageScriptIndex value: 1 @@ -4404,6 +4340,10 @@ PrefabInstance: propertyPath: use_saved_meshes value: 1 objectReference: {fileID: 0} + - target: {fileID: 1137478775515482002, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: spotObserverClient + value: + objectReference: {fileID: 971801071} - target: {fileID: 1137478775531494915, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: RosBridgeServerUrl value: ws://128.148.138.132:9090 @@ -5031,7 +4971,11 @@ PrefabInstance: m_RemovedComponents: - {fileID: 5143370623281230838, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} - {fileID: 1137478774082161501, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} - m_RemovedGameObjects: [] + m_RemovedGameObjects: + - {fileID: 1137478774821249053, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + - {fileID: 1137478774429119418, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + - {fileID: 1137478775531494912, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + - {fileID: 1137478773798299892, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} m_AddedGameObjects: - targetCorrespondingSourceObject: {fileID: 4598551966875127983, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} insertIndex: -1 @@ -5042,18 +4986,15 @@ PrefabInstance: - targetCorrespondingSourceObject: {fileID: 1137478775229705932, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} insertIndex: 0 addedObject: {fileID: 164802371} + - targetCorrespondingSourceObject: {fileID: 1137478774763210852, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + insertIndex: -1 + addedObject: {fileID: 971801070} - targetCorrespondingSourceObject: {fileID: 1137478774082161500, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} insertIndex: 1 addedObject: {fileID: 312514186} - targetCorrespondingSourceObject: {fileID: 5050333314984075172, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} insertIndex: 0 addedObject: {fileID: 804843770} - - targetCorrespondingSourceObject: {fileID: 1137478774281308966, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} - insertIndex: 4 - addedObject: {fileID: 128476701} - - targetCorrespondingSourceObject: {fileID: 1137478774281308966, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} - insertIndex: 5 - addedObject: {fileID: 1953341853} - targetCorrespondingSourceObject: {fileID: 3310871052953474555, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} insertIndex: 0 addedObject: {fileID: 1834046043} @@ -5740,6 +5681,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -5761,6 +5704,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -6279,6 +6223,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -6301,6 +6247,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -7099,7 +7046,7 @@ MonoBehaviour: another_manager: {fileID: 1639569377} CVDLeft: {fileID: 305849529} CVDRight: {fileID: 1814602523} - activate_depth_estimation: 1 + activate_depth_estimation: 0 activate_ICP: 0 activate_edge_detection: 1 edgeThreshold: 21 @@ -7307,8 +7254,8 @@ MonoBehaviour: gripper: {fileID: 669083966} generalControls: {fileID: 1834046044} depthSubscribers: - - {fileID: 941303697} - - {fileID: 810543855} + - {fileID: 0} + - {fileID: 0} armBase: {fileID: 1785486167} ghostArm: {fileID: 1988292211} ghostFinger: {fileID: 2021508742} @@ -8028,6 +7975,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -8049,6 +7998,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -8126,11 +8076,11 @@ MonoBehaviour: drive: {fileID: 616808317} syncDrive: {fileID: 0} depthSubscribers: - - {fileID: 810543855} - - {fileID: 941303697} + - {fileID: 0} + - {fileID: 0} colorSubscribers: - - {fileID: 260625234} - - {fileID: 383063648} + - {fileID: 0} + - {fileID: 0} odometrySubscriber: {fileID: 693475762} pointClouds: - {fileID: 409472603} @@ -8291,6 +8241,8 @@ MeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -8312,6 +8264,7 @@ MeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -8435,6 +8388,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -8456,6 +8411,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -8690,6 +8646,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -8711,6 +8669,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -8882,6 +8841,8 @@ MeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -8903,6 +8864,7 @@ MeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -8927,7 +8889,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: af1357cb9840a354fb4227c3fd532d54, type: 3} m_Name: m_EditorClassIdentifier: - colorSubscriber: {fileID: 1953341855} + colorSubscriber: {fileID: 0} material: {fileID: 2100000, guid: 82e05df61dda7c4499ce62b04812faff, type: 2} use_saved_meshes: 0 --- !u!114 &804843775 @@ -8967,17 +8929,6 @@ MonoBehaviour: Serializer: 0 protocol: 0 RosBridgeServerUrl: ---- !u!114 &810543855 stripped -MonoBehaviour: - m_CorrespondingSourceObject: {fileID: 1137478774429119364, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} - m_PrefabInstance: {fileID: 409472599} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 0} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: f88981aeb3483944785bdbed96c04bf4, type: 3} - m_Name: - m_EditorClassIdentifier: --- !u!1 &819668449 GameObject: m_ObjectHideFlags: 0 @@ -9620,17 +9571,6 @@ Transform: m_Children: [] m_Father: {fileID: 1085138239} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} ---- !u!114 &941303697 stripped -MonoBehaviour: - m_CorrespondingSourceObject: {fileID: 1137478773798299894, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} - m_PrefabInstance: {fileID: 409472599} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 0} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: f88981aeb3483944785bdbed96c04bf4, type: 3} - m_Name: - m_EditorClassIdentifier: --- !u!1 &947366004 GameObject: m_ObjectHideFlags: 0 @@ -9693,8 +9633,8 @@ MonoBehaviour: gripper: {fileID: 693475758} generalControls: {fileID: 164802372} depthSubscribers: - - {fileID: 941303697} - - {fileID: 810543855} + - {fileID: 0} + - {fileID: 0} armBase: {fileID: 1576356864} ghostArm: {fileID: 293419650} ghostFinger: {fileID: 498688831} @@ -9971,6 +9911,54 @@ Transform: m_Children: [] m_Father: {fileID: 493375339} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!1 &971801069 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 971801070} + - component: {fileID: 971801071} + m_Layer: 0 + m_Name: SpotObserverClient + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!4 &971801070 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 971801069} + serializedVersion: 2 + m_LocalRotation: {x: -0, y: -0, z: -0, w: 1} + m_LocalPosition: {x: 0.060298983, y: 0.042863965, z: -0.43645263} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 1475223667} + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!114 &971801071 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 971801069} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: cdc798fc4b250c04b91b6a0767d25d31, type: 3} + m_Name: + m_EditorClassIdentifier: + RobotIP: 128.148.140.22 + username: user + password: bigbubbabigbubba + EnableDebugDumps: 0 --- !u!1 &973382266 GameObject: m_ObjectHideFlags: 0 @@ -11314,6 +11302,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -11335,6 +11325,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -11440,6 +11431,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -11461,6 +11454,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -11535,6 +11529,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -11556,6 +11552,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -12059,6 +12056,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -12080,6 +12079,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -12157,11 +12157,11 @@ MonoBehaviour: drive: {fileID: 693475761} syncDrive: {fileID: 2123763400} depthSubscribers: - - {fileID: 810543855} - - {fileID: 941303697} + - {fileID: 0} + - {fileID: 0} colorSubscribers: - - {fileID: 260625234} - - {fileID: 383063648} + - {fileID: 0} + - {fileID: 0} odometrySubscriber: {fileID: 693475762} pointClouds: - {fileID: 326679877} @@ -12299,6 +12299,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -12320,6 +12322,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -13034,6 +13037,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -13055,6 +13060,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -13273,6 +13279,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -13294,6 +13302,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -13430,6 +13439,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -13452,6 +13463,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -13536,6 +13548,11 @@ Transform: m_Children: [] m_Father: {fileID: 437099332} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!4 &1475223667 stripped +Transform: + m_CorrespondingSourceObject: {fileID: 1137478774763210852, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + m_PrefabInstance: {fileID: 409472599} + m_PrefabAsset: {fileID: 0} --- !u!1 &1480700625 GameObject: m_ObjectHideFlags: 0 @@ -13754,6 +13771,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -13775,6 +13794,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -13855,6 +13875,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -13876,6 +13898,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -14154,7 +14177,7 @@ MonoBehaviour: TimeStep: 0 meshRenderer: {fileID: 43937879} texture2D: {fileID: 0} - associatedDepth: {fileID: 810543855} + associatedDepth: {fileID: 0} printRate: 0 printSyncTime: 0 printClosestTime: 0 @@ -14279,6 +14302,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -14300,6 +14325,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -14445,6 +14471,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -14466,6 +14494,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -14610,6 +14639,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -14631,6 +14662,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -14908,6 +14940,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -14929,6 +14963,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -15075,6 +15110,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -15096,6 +15133,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -16019,6 +16057,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -16040,6 +16080,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -16177,6 +16218,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -16198,6 +16241,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -17077,6 +17121,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -17098,6 +17144,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -17221,6 +17268,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -17242,6 +17291,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -17338,76 +17388,6 @@ Transform: m_CorrespondingSourceObject: {fileID: 1137478773629964866, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} m_PrefabInstance: {fileID: 409472599} m_PrefabAsset: {fileID: 0} ---- !u!1 &1953341852 -GameObject: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - serializedVersion: 6 - m_Component: - - component: {fileID: 1953341853} - - component: {fileID: 1953341855} - - component: {fileID: 1953341854} - m_Layer: 0 - m_Name: HandColorSubscriber - m_TagString: Untagged - m_Icon: {fileID: 0} - m_NavMeshLayer: 0 - m_StaticEditorFlags: 0 - m_IsActive: 1 ---- !u!4 &1953341853 -Transform: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 1953341852} - serializedVersion: 2 - m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} - m_LocalPosition: {x: 0, y: 0, z: 0} - m_LocalScale: {x: 1, y: 1, z: 1} - m_ConstrainProportionsScale: 0 - m_Children: [] - m_Father: {fileID: 43937878} - m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} ---- !u!114 &1953341854 -MonoBehaviour: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 1953341852} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: ff98778ae051cfd40a6e4effdf851ebd, type: 3} - m_Name: - m_EditorClassIdentifier: - SecondsTimeout: 10 - Serializer: 0 - protocol: 1 - RosBridgeServerUrl: ws://128.148.138.132:9090 ---- !u!114 &1953341855 -MonoBehaviour: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 1953341852} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: 441fcddae936d184f8196aeed41a9f18, type: 3} - m_Name: - m_EditorClassIdentifier: - Topic: /spot/stream_image/hand_color_image/image - TimeStep: 0 - meshRenderer: {fileID: 43937879} - texture2D: {fileID: 0} - associatedDepth: {fileID: 0} - printRate: 0 - printSyncTime: 1 - printClosestTime: 0 - latest_time: 0 --- !u!1 &1955072827 GameObject: m_ObjectHideFlags: 0 @@ -18137,6 +18117,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -18158,6 +18140,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 @@ -18350,6 +18333,8 @@ SkinnedMeshRenderer: m_RayTracingAccelStructBuildFlagsOverride: 0 m_RayTracingAccelStructBuildFlags: 1 m_SmallMeshCulling: 1 + m_ForceMeshLod: -1 + m_MeshLodSelectionBias: 0 m_RenderingLayerMask: 1 m_RendererPriority: 0 m_Materials: @@ -18371,6 +18356,7 @@ SkinnedMeshRenderer: m_AutoUVMaxDistance: 0.5 m_AutoUVMaxAngle: 89 m_LightmapParameters: {fileID: 0} + m_GlobalIlluminationMeshLod: 0 m_SortingLayerID: 0 m_SortingLayer: 0 m_SortingOrder: 0 diff --git a/Assets/Scripts/DrawMeshInstanced.cs b/Assets/Scripts/DrawMeshInstanced.cs index ace490c..4dc0432 100644 --- a/Assets/Scripts/DrawMeshInstanced.cs +++ b/Assets/Scripts/DrawMeshInstanced.cs @@ -40,6 +40,8 @@ public class DrawMeshInstanced : MonoBehaviour public RawImageSubscriber depthSubscriber; // ROS subscriber that holds the depth array public JPEGImageSubscriber colorSubscriber; // ROS subscriber holding the color image + public SpotObserverClient spotObserverClient; + public bool savePointCloud; // allow user to save point cloud public ComputeShader compute; @@ -296,7 +298,6 @@ private void InitializeBuffers() // Initialize buffer with the given population. MeshProperties[] properties = new MeshProperties[population]; - meshPropertiesBuffer = new ComputeBuffer((int)population, MeshProperties.Size()); meshPropertiesBuffer.SetData(GetProperties()); @@ -396,17 +397,35 @@ private void UpdateTexture() if (use_saved_meshes) { + Debug.Log("Using saved mesh"); for (int i = 0; i < 480 * 640; i++) { - depth_ar[i] = depth_ar_saved[i]; + //depth_ar[i] = depth_ar_saved[i]; + depth_ar[i] = 1.0f; } + + (color_image, depth_image) = spotObserverClient.GetCameraFeeds(0); + Debug.Log("Color image: " + (color_image == null ? "null" : color_image.width + "x" + color_image.height)); + + //if (depth_image != null) + //{ + // depth_ar = depth_image.GetPixelData(0).ToArray(); + //} + new_depth_to_render = true; } else { - DestroyImmediate(color_image, true); - color_image = copy_texture(colorSubscriber.texture2D); - depth_ar = depthSubscriber.getDepthArr(); + Debug.Log("Getting camera feeds from Spot Observer Client"); + (color_image, depth_image) = spotObserverClient.GetCameraFeeds(0); + Debug.Log("Color image: " + (color_image == null ? "null" : color_image.width + "x" + color_image.height)); + + if (depth_image != null) + { + depth_ar = depth_image.GetPixelData(0).ToArray(); + } + Debug.Log("Depth image: " + (depth_image == null ? "null" : depth_image.width + "x" + depth_image.height)); + //depth_ar = depthSubscriber.getDepthArr(); if (depthSubscriber.new_depth == true) { new_depth_to_render = true; @@ -414,6 +433,7 @@ private void UpdateTexture() depthSubscriber.new_depth = false; } + Debug.Log("Depth array length: " + depth_ar.Length); if (depth_ar.Length < 640 * 480) { return; @@ -487,6 +507,7 @@ private void Update() return; } + Debug.Log("UPDATE CALLED"); UpdateTexture(); diff --git a/Assets/Scripts/SpotObserverClient.cs b/Assets/Scripts/SpotObserverClient.cs new file mode 100644 index 0000000..8c691b5 --- /dev/null +++ b/Assets/Scripts/SpotObserverClient.cs @@ -0,0 +1,228 @@ +using RosSharp.RosBridgeClient.MessageTypes.Std; +using System; +using System.Runtime.InteropServices; +using Unity.InferenceEngine; + +using UnityEngine; + + +public class SpotObserverClient : MonoBehaviour +{ + + [DllImport("SpotObserverLib", CharSet = CharSet.Ansi)] + private static extern int SOb_ConnectToSpot( + string robot_ip, + string username, + string password + ); + + [DllImport("SpotObserverLib")] + private static extern bool SOb_DisconnectFromSpot(int robot_id); + + [DllImport("SpotObserverLib", CharSet = CharSet.Ansi)] + private static extern bool SOb_ReadCameraFeeds( + int robot_id, + uint camera_mask + ); + + [DllImport("SpotObserverLib")] + private static extern bool SOb_RegisterUnityReadbackBuffers( + int robot_id, + uint cam_bit, + IntPtr rgb_resource, + IntPtr depth_resource, + int img_buffer_size, // In bytes + int depth_buffer_size // In bytes + ); + + [DllImport("SpotObserverLib")] + private static extern bool SOb_PushNextImageSetToUnityBuffers( + int robot_id + ); + + [DllImport("SpotObserverLib")] + private static extern bool SOb_SetUnityLogCallback( + LogCallback callback + ); + + [DllImport("SpotObserverLib", CharSet = CharSet.Ansi)] + private static extern void SOb_ToggleDebugDumps( + string dump_path + ); + + // Make sure the enum matches the C++ enum in spot-observer.h + public enum SpotCamera + { + BACK = 0x1, + FRONTLEFT = 0x2, + FRONTRIGHT = 0x4, + LEFT = 0x8, + RIGHT = 0x10, + HAND = 0x20, + NUM_CAMERAS = 0x40, + }; + + private delegate void LogCallback(string message); + + private static void PluginLogCallback(string message) + { + Debug.Log("[SpotObserverLib] " + message); + } + + public string RobotIP; + public string username; + public string password; + + public bool EnableDebugDumps = false; + + private int robot_id = -1; + private bool isConnected = false; + + private Texture2D[] rgb_textures; + private Texture2D[] depth_textures; + + //private Tensor[] rgb_tensors; + //private Tensor[] depth_tensors; + + private IntPtr[] rgb_resources; + private IntPtr[] depth_resources; + + + void Start() + { + SOb_SetUnityLogCallback(PluginLogCallback); + RobotIP = RobotIP.Trim(); + robot_id = SOb_ConnectToSpot(RobotIP, username, password); + if (robot_id < 0) + { + Debug.LogError("Failed to connect to Spot robot at " + RobotIP); + isConnected = false; + return; + } + Debug.Log("Connected to Spot robot at " + RobotIP); + isConnected = true; + + if (EnableDebugDumps) + { + SOb_ToggleDebugDumps("SPOT_OBSERVER_DUMPS"); + } + + // Initialize camera resources + // TODO: Unhardcode shapes + TensorShape rgb_shape = new TensorShape(1, 3, 480, 640); // RGB shape + TensorShape depth_shape = new TensorShape(1, 1, 480, 640); // Depth shape + + // Initialize textures and tensors for each camera + rgb_textures = new Texture2D[3]; + depth_textures = new Texture2D[3]; + rgb_resources = new IntPtr[3]; + depth_resources = new IntPtr[3]; + + //rgb_tensors = new Tensor[] + //{ + // new Tensor(rgb_shape), + // new Tensor(rgb_shape), + // new Tensor(rgb_shape) + //}; + //depth_tensors = new Tensor[] + //{ + // new Tensor(depth_shape), + // new Tensor(depth_shape), + // new Tensor(depth_shape) + //}; + + uint[] cams = { + (uint)SpotCamera.FRONTRIGHT, + (uint)SpotCamera.FRONTLEFT, + (uint)SpotCamera.HAND, + + }; + + uint all_cams = 0; + for (var i = 0; i < rgb_textures.Length; i++) + { + rgb_textures[i] = new Texture2D(640, 480, TextureFormat.RGB24, false); + depth_textures[i] = new Texture2D(640, 480, TextureFormat.RFloat, false); + rgb_resources[i] = rgb_textures[i].GetNativeTexturePtr(); + depth_resources[i] = depth_textures[i].GetNativeTexturePtr(); + + //rgb_resources[i] = ComputeTensorData.Pin(rgb_tensors[i]).buffer.GetNativeBufferPtr(); + //depth_resources[i] = ComputeTensorData.Pin(depth_tensors[i]).buffer.GetNativeBufferPtr(); + + // Register textures with the Spot observer + if (!SOb_RegisterUnityReadbackBuffers(robot_id, cams[i], rgb_resources[i], depth_resources[i], 640 * 480 * 3, 640 * 480 * 4)) + { + Debug.LogError("Failed to register textures for camera " + cams[i]); + } + + all_cams |= cams[i]; + } + + // Kick off reading camera feeds + if (!SOb_ReadCameraFeeds(robot_id, all_cams)) + { + Debug.LogError("Failed to read camera feeds from Spot robot."); + isConnected = false; + return; + } + + if (!SOb_PushNextImageSetToUnityBuffers(robot_id)) + { + //Debug.LogError("Failed to read back image set to Unity buffers."); + return; + } + Debug.Log("Successfully registered and read camera feeds from Spot robot."); + } + + void OnDestroy() + { + if (isConnected) + { + SOb_DisconnectFromSpot(robot_id); + Debug.Log("Disconnected from Spot robot."); + } + //// Dispose tensors + //foreach (var tensor in rgb_tensors) + //{ + // tensor.Dispose(); + //} + //foreach (var tensor in depth_tensors) + //{ + // tensor.Dispose(); + //} + foreach (var texture in rgb_textures) + { + if (texture != null) + { + Destroy(texture); + } + } + } + + void Update() + { + if (robot_id < 0) + { + Debug.LogError("Not connected to Spot robot. Cannot update."); + return; + } + + if (!SOb_PushNextImageSetToUnityBuffers(robot_id)) + { + return; + } + + Debug.LogWarning("SUCCESSFULLY READ CAMERA FEEDS"); + } + + public (Texture2D, Texture2D) GetCameraFeeds(int id) + { + if (!isConnected) + { + Debug.LogError("Not connected to Spot robot. Cannot get camera feeds."); + return (null, null); + } + Debug.LogWarning("Returning RGB and Depth for " + id); + return (rgb_textures[id], depth_textures[id]); + } +} diff --git a/Assets/Scripts/SpotObserverClient.cs.meta b/Assets/Scripts/SpotObserverClient.cs.meta new file mode 100644 index 0000000..52f185b --- /dev/null +++ b/Assets/Scripts/SpotObserverClient.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: cdc798fc4b250c04b91b6a0767d25d31 \ No newline at end of file diff --git a/Assets/Scripts/UBModel.cs b/Assets/Scripts/UBModel.cs index e0497c7..0a9a396 100644 --- a/Assets/Scripts/UBModel.cs +++ b/Assets/Scripts/UBModel.cs @@ -1,9 +1,7 @@ using System; -using System.Collections; -using System.Collections.Generic; using System.Runtime.InteropServices; -using Unity.Sentis; -using UnityEditor; +using Unity.InferenceEngine; + using UnityEngine; @@ -14,7 +12,7 @@ public class UBModel : MonoBehaviour private static extern bool UB_LoadModel(string modelPath, string backend); [DllImport("UnityBYOM")] private static extern bool UB_RunInference( - IntPtr texResource, // ID3D12Resource* of the Unity texture + IntPtr texResource, // ID3D12Resource* of the Unity tensor IntPtr depthResource, // another ID3D12Resource* IntPtr outputBuffer, // ComputeBuffer.GetNativeBufferPtr() int inputCount, // Number of input elements diff --git a/Assets/Urdf/default/spot_description/meshes/body.dae.meta b/Assets/Urdf/default/spot_description/meshes/body.dae.meta index 5fb4463..04048d5 100644 --- a/Assets/Urdf/default/spot_description/meshes/body.dae.meta +++ b/Assets/Urdf/default/spot_description/meshes/body.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: 33d04cdb7d83c6644b761088efbdea09 ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/meshes/front_left_hip.dae.meta b/Assets/Urdf/default/spot_description/meshes/front_left_hip.dae.meta index aec7f12..55a4ed1 100644 --- a/Assets/Urdf/default/spot_description/meshes/front_left_hip.dae.meta +++ b/Assets/Urdf/default/spot_description/meshes/front_left_hip.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: b4565630e9f4d9040ac488b23fd43620 ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/meshes/front_left_lower_leg.dae.meta b/Assets/Urdf/default/spot_description/meshes/front_left_lower_leg.dae.meta index e0dc31c..ff6e8a6 100644 --- a/Assets/Urdf/default/spot_description/meshes/front_left_lower_leg.dae.meta +++ b/Assets/Urdf/default/spot_description/meshes/front_left_lower_leg.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: 0768174edffea91409ee4bccd12a6b18 ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/meshes/front_left_upper_leg.dae.meta b/Assets/Urdf/default/spot_description/meshes/front_left_upper_leg.dae.meta index 13065bf..7ab24bb 100644 --- a/Assets/Urdf/default/spot_description/meshes/front_left_upper_leg.dae.meta +++ b/Assets/Urdf/default/spot_description/meshes/front_left_upper_leg.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: 6aac39cf07429d14fb797365907e218d ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/meshes/front_right_hip.dae.meta b/Assets/Urdf/default/spot_description/meshes/front_right_hip.dae.meta index 3685222..12c0779 100644 --- a/Assets/Urdf/default/spot_description/meshes/front_right_hip.dae.meta +++ b/Assets/Urdf/default/spot_description/meshes/front_right_hip.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: 287fac4076716e14a8f961aba228c439 ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/meshes/front_right_lower_leg.dae.meta b/Assets/Urdf/default/spot_description/meshes/front_right_lower_leg.dae.meta index 88d73b4..b9cd22f 100644 --- a/Assets/Urdf/default/spot_description/meshes/front_right_lower_leg.dae.meta +++ b/Assets/Urdf/default/spot_description/meshes/front_right_lower_leg.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: 19a4ead02cb217a4fb4efb766081cade ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/meshes/front_right_upper_leg.dae.meta b/Assets/Urdf/default/spot_description/meshes/front_right_upper_leg.dae.meta index bcfb85e..d820648 100644 --- a/Assets/Urdf/default/spot_description/meshes/front_right_upper_leg.dae.meta +++ b/Assets/Urdf/default/spot_description/meshes/front_right_upper_leg.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: 276fea83617a3d1459dc4986692954f3 ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/meshes/rear_left_hip.dae.meta b/Assets/Urdf/default/spot_description/meshes/rear_left_hip.dae.meta index 058e625..a27e4dd 100644 --- a/Assets/Urdf/default/spot_description/meshes/rear_left_hip.dae.meta +++ b/Assets/Urdf/default/spot_description/meshes/rear_left_hip.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: b8aa99e248e2a32458da77460f284199 ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/meshes/rear_left_lower_leg.dae.meta b/Assets/Urdf/default/spot_description/meshes/rear_left_lower_leg.dae.meta index 7393752..36110bc 100644 --- a/Assets/Urdf/default/spot_description/meshes/rear_left_lower_leg.dae.meta +++ b/Assets/Urdf/default/spot_description/meshes/rear_left_lower_leg.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: 626b86b8841d00f4898cdc22a4391a56 ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/meshes/rear_left_upper_leg.dae.meta b/Assets/Urdf/default/spot_description/meshes/rear_left_upper_leg.dae.meta index 60a0861..ddd249a 100644 --- a/Assets/Urdf/default/spot_description/meshes/rear_left_upper_leg.dae.meta +++ b/Assets/Urdf/default/spot_description/meshes/rear_left_upper_leg.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: f518ea4b9c976ab46b9e1785d79b616c ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/meshes/rear_right_hip.dae.meta b/Assets/Urdf/default/spot_description/meshes/rear_right_hip.dae.meta index c9cc738..dc02618 100644 --- a/Assets/Urdf/default/spot_description/meshes/rear_right_hip.dae.meta +++ b/Assets/Urdf/default/spot_description/meshes/rear_right_hip.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: 473c15c61fbe01a45b73bae3e9c5c40d ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/meshes/rear_right_lower_leg.dae.meta b/Assets/Urdf/default/spot_description/meshes/rear_right_lower_leg.dae.meta index d0b79f2..8ca4508 100644 --- a/Assets/Urdf/default/spot_description/meshes/rear_right_lower_leg.dae.meta +++ b/Assets/Urdf/default/spot_description/meshes/rear_right_lower_leg.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: 62ce33c985f870e4c9f077daf1ba48e8 ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/meshes/rear_right_upper_leg.dae.meta b/Assets/Urdf/default/spot_description/meshes/rear_right_upper_leg.dae.meta index 9ac7de0..ab21388 100644 --- a/Assets/Urdf/default/spot_description/meshes/rear_right_upper_leg.dae.meta +++ b/Assets/Urdf/default/spot_description/meshes/rear_right_upper_leg.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: b38ee6c069cba404cb608f8dceec0b9a ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_el0.dae.meta b/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_el0.dae.meta index bca9cd9..18f124a 100644 --- a/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_el0.dae.meta +++ b/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_el0.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: 227990b11a9bb4049980e5914768578f ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_el1.dae.meta b/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_el1.dae.meta index 9ffb04d..1c7154c 100644 --- a/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_el1.dae.meta +++ b/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_el1.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: 9057163d25f2a5f4e99552766e02be01 ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_fngr.dae.meta b/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_fngr.dae.meta index 29abbf7..ef2ce95 100644 --- a/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_fngr.dae.meta +++ b/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_fngr.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: 3f1bce988d51f074592efa857c73ec35 ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_hr0.dae.meta b/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_hr0.dae.meta index ef2e73c..4721402 100644 --- a/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_hr0.dae.meta +++ b/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_hr0.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: a220b67430ddebf47b91d2ce9a7564e6 ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_sh0.dae.meta b/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_sh0.dae.meta index 1fe03f8..cbd3cca 100644 --- a/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_sh0.dae.meta +++ b/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_sh0.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: 47e448ab3fda45444bdcc2232218cf37 ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_sh1.dae.meta b/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_sh1.dae.meta index 63200b7..d00e6f8 100644 --- a/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_sh1.dae.meta +++ b/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_sh1.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: 12ad25adac320d6479849d6134ebd886 ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_wr0.dae.meta b/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_wr0.dae.meta index 9919b52..b3ddaed 100644 --- a/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_wr0.dae.meta +++ b/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_wr0.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: 35a8781b547df9044b7fd7848d4d54fe ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_wr1.dae.meta b/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_wr1.dae.meta index b265889..66a2a2a 100644 --- a/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_wr1.dae.meta +++ b/Assets/Urdf/default/spot_description/spot/meshes/arm0_link_wr1.dae.meta @@ -1,7 +1,7 @@ fileFormatVersion: 2 guid: afcc5b4344792e64b9603b47353a75f0 ModelImporter: - serializedVersion: 22200 + serializedVersion: 24200 internalIDToNameTable: [] externalObjects: {} materials: @@ -56,6 +56,9 @@ ModelImporter: maxBonesPerVertex: 4 minBoneWeight: 0.001 optimizeBones: 1 + generateMeshLods: 0 + meshLodGenerationFlags: 0 + maximumMeshLod: -1 meshOptimizationFlags: -1 indexFormat: 0 secondaryUVAngleDistortion: 8 diff --git a/Assets/XR/Settings/OpenXR Package Settings.asset b/Assets/XR/Settings/OpenXR Package Settings.asset index 1469c77..7c7b2eb 100644 --- a/Assets/XR/Settings/OpenXR Package Settings.asset +++ b/Assets/XR/Settings/OpenXR Package Settings.asset @@ -117,6 +117,7 @@ MonoBehaviour: version: 0.1.0 featureIdInternal: com.unity.openxr.feature.arfoundation-meta-anchor openxrExtensionStrings: XR_FB_spatial_entity XR_META_spatial_entity_discovery XR_META_spatial_entity_persistence + XR_META_spatial_entity_sharing XR_META_spatial_entity_group_sharing XR_FB_spatial_entity_query company: Unity Technologies priority: 0 required: 0 @@ -140,6 +141,28 @@ MonoBehaviour: company: Meta priority: 0 required: 0 +--- !u!114 &-5975233314077489818 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 6096094d69d82cc40947df806f66ab34, type: 3} + m_Name: OpenXRCompositionLayersFeature Android + m_EditorClassIdentifier: Unity.XR.OpenXR::UnityEngine.XR.OpenXR.Features.CompositionLayers.OpenXRCompositionLayersFeature + m_enabled: 0 + nameUi: Composition Layers Support + version: 1.0.0 + featureIdInternal: com.unity.openxr.feature.compositionlayers + openxrExtensionStrings: XR_KHR_composition_layer_cylinder XR_KHR_composition_layer_equirect + XR_KHR_composition_layer_equirect2 XR_KHR_composition_layer_cube XR_KHR_composition_layer_color_scale_bias + XR_KHR_android_surface_swapchain + company: Unity + priority: 0 + required: 0 --- !u!114 &-5398722598577384755 MonoBehaviour: m_ObjectHideFlags: 0 @@ -226,6 +249,28 @@ MonoBehaviour: company: Meta priority: 0 required: 0 +--- !u!114 &-4571090997830837056 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 6096094d69d82cc40947df806f66ab34, type: 3} + m_Name: OpenXRCompositionLayersFeature Standalone + m_EditorClassIdentifier: Unity.XR.OpenXR::UnityEngine.XR.OpenXR.Features.CompositionLayers.OpenXRCompositionLayersFeature + m_enabled: 1 + nameUi: Composition Layers Support + version: 1.0.0 + featureIdInternal: com.unity.openxr.feature.compositionlayers + openxrExtensionStrings: XR_KHR_composition_layer_cylinder XR_KHR_composition_layer_equirect + XR_KHR_composition_layer_equirect2 XR_KHR_composition_layer_cube XR_KHR_composition_layer_color_scale_bias + XR_KHR_android_surface_swapchain + company: Unity + priority: 0 + required: 0 --- !u!114 &-4247736531263632017 MonoBehaviour: m_ObjectHideFlags: 0 @@ -242,7 +287,8 @@ MonoBehaviour: nameUi: 'Meta Quest: Planes' version: 0.1.0 featureIdInternal: com.unity.openxr.feature.arfoundation-meta-plane - openxrExtensionStrings: XR_FB_scene XR_FB_spatial_entity XR_META_spatial_entity_discovery + openxrExtensionStrings: XR_FB_scene XR_FB_spatial_entity XR_FB_spatial_entity_container + XR_META_spatial_entity_discovery company: Unity Technologies priority: 0 required: 0 @@ -287,6 +333,27 @@ MonoBehaviour: company: Unity Technologies priority: 0 required: 0 +--- !u!114 &-3566507896945661697 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 01512f11fe5096443ad972feb69271f6, type: 3} + m_Name: SpaceWarpFeature Android + m_EditorClassIdentifier: Unity.XR.OpenXR::UnityEngine.XR.OpenXR.Features.SpaceWarpFeature + m_enabled: 0 + nameUi: Application SpaceWarp + version: 1.0.0 + featureIdInternal: com.unity.openxr.feature.spacewarp + openxrExtensionStrings: XR_FB_space_warp + company: Unity + priority: 0 + required: 0 + m_UseRightHandedNDC: 0 --- !u!114 &-2987421080943291414 MonoBehaviour: m_ObjectHideFlags: 0 @@ -397,6 +464,7 @@ MonoBehaviour: version: 0.1.0 featureIdInternal: com.unity.openxr.feature.arfoundation-meta-anchor openxrExtensionStrings: XR_FB_spatial_entity XR_META_spatial_entity_discovery XR_META_spatial_entity_persistence + XR_META_spatial_entity_sharing XR_META_spatial_entity_group_sharing XR_FB_spatial_entity_query company: Unity Technologies priority: 0 required: 0 @@ -548,6 +616,7 @@ MonoBehaviour: m_EditorClassIdentifier: features: - {fileID: 7438672597232742020} + - {fileID: -4571090997830837056} - {fileID: -449741914172589832} - {fileID: -1992737726343282275} - {fileID: 3973412703780260892} @@ -563,6 +632,7 @@ MonoBehaviour: - {fileID: -177000443299484431} - {fileID: 8827381569515383560} - {fileID: 2530328125348231399} + - {fileID: -594876591609520937} - {fileID: 4240071139046573350} - {fileID: -3820250420026720182} - {fileID: 2960767094789127846} @@ -582,6 +652,7 @@ MonoBehaviour: - {fileID: -1259320862161519326} - {fileID: -8638663112018935848} m_renderMode: 0 + m_latencyOptimization: 0 m_autoColorSubmissionMode: 1 m_colorSubmissionModes: m_List: 00000000 @@ -590,7 +661,30 @@ MonoBehaviour: m_optimizeBufferDiscards: 0 m_symmetricProjection: 0 m_optimizeMultiviewRenderRegions: 0 + m_multiviewRenderRegionsOptimizationMode: 0 + m_hasMigratedMultiviewRenderRegionSetting: 1 m_foveatedRenderingApi: 0 + m_useOpenXRPredictedTime: 0 +--- !u!114 &-594876591609520937 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 651d103057d74c4d8e51d0b368d3d98b, type: 3} + m_Name: ColocationDiscoveryFeature Standalone + m_EditorClassIdentifier: Unity.XR.MetaOpenXR::UnityEngine.XR.OpenXR.Features.Meta.ColocationDiscoveryFeature + m_enabled: 0 + nameUi: 'Meta Quest: Colocation Discovery' + version: 1.0.0 + featureIdInternal: com.unity.openxr.feature.meta-colocation-discovery + openxrExtensionStrings: XR_META_colocation_discovery + company: Unity Technologies + priority: 0 + required: 0 --- !u!114 &-555646166468275018 MonoBehaviour: m_ObjectHideFlags: 0 @@ -673,6 +767,7 @@ MonoBehaviour: company: Unity Technologies priority: 0 required: 0 + m_SuppressVisibility: 0 --- !u!114 &11400000 MonoBehaviour: m_ObjectHideFlags: 0 @@ -733,6 +828,8 @@ MonoBehaviour: lateLatchingMode: 0 lateLatchingDebug: 0 optimizeMultiviewRenderRegions: 0 + multiviewRenderRegionsOptimizationMode: 0 + m_hasMigratedMultiviewRenderRegions: 1 spacewarpMotionVectorTextureFormat: 0 --- !u!114 &156886475746385215 MonoBehaviour: @@ -956,7 +1053,8 @@ MonoBehaviour: nameUi: 'Meta Quest: Planes' version: 0.1.0 featureIdInternal: com.unity.openxr.feature.arfoundation-meta-plane - openxrExtensionStrings: XR_FB_scene XR_FB_spatial_entity XR_META_spatial_entity_discovery + openxrExtensionStrings: XR_FB_scene XR_FB_spatial_entity XR_FB_spatial_entity_container + XR_META_spatial_entity_discovery company: Unity Technologies priority: 0 required: 0 @@ -996,7 +1094,8 @@ MonoBehaviour: nameUi: 'Meta Quest: Bounding Boxes' version: 0.1.0 featureIdInternal: com.unity.openxr.feature.arfoundation-meta-bounding-boxes - openxrExtensionStrings: XR_FB_scene XR_FB_spatial_entity XR_META_spatial_entity_discovery + openxrExtensionStrings: XR_FB_scene XR_FB_spatial_entity XR_FB_spatial_entity_container + XR_META_spatial_entity_discovery company: Unity Technologies priority: 0 required: 0 @@ -1344,6 +1443,8 @@ MonoBehaviour: m_EditorClassIdentifier: features: - {fileID: -5158844391086987368} + - {fileID: -3566507896945661697} + - {fileID: -5975233314077489818} - {fileID: -1819930393743201246} - {fileID: -1201980078073350488} - {fileID: 7305062941160009587} @@ -1358,6 +1459,7 @@ MonoBehaviour: - {fileID: 7273668044344639028} - {fileID: 1653993578541201969} - {fileID: 4507826202002996448} + - {fileID: 8130131695087115765} - {fileID: 1076638229118490786} - {fileID: 7422930540860876341} - {fileID: -1823111745317354088} @@ -1377,6 +1479,7 @@ MonoBehaviour: - {fileID: 5187790130119334493} - {fileID: -2496776499858408549} m_renderMode: 1 + m_latencyOptimization: 0 m_autoColorSubmissionMode: 1 m_colorSubmissionModes: m_List: 00000000 @@ -1385,7 +1488,10 @@ MonoBehaviour: m_optimizeBufferDiscards: 0 m_symmetricProjection: 0 m_optimizeMultiviewRenderRegions: 0 + m_multiviewRenderRegionsOptimizationMode: 0 + m_hasMigratedMultiviewRenderRegionSetting: 1 m_foveatedRenderingApi: 0 + m_useOpenXRPredictedTime: 0 --- !u!114 &6199120310985651292 MonoBehaviour: m_ObjectHideFlags: 0 @@ -1426,6 +1532,7 @@ MonoBehaviour: company: Unity Technologies priority: 0 required: 0 + m_SuppressVisibility: 0 --- !u!114 &7305062941160009587 MonoBehaviour: m_ObjectHideFlags: 0 @@ -1507,6 +1614,26 @@ MonoBehaviour: company: Unity Technologies priority: 2147483647 required: 0 +--- !u!114 &8130131695087115765 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 651d103057d74c4d8e51d0b368d3d98b, type: 3} + m_Name: ColocationDiscoveryFeature Android + m_EditorClassIdentifier: Unity.XR.MetaOpenXR::UnityEngine.XR.OpenXR.Features.Meta.ColocationDiscoveryFeature + m_enabled: 0 + nameUi: 'Meta Quest: Colocation Discovery' + version: 1.0.0 + featureIdInternal: com.unity.openxr.feature.meta-colocation-discovery + openxrExtensionStrings: XR_META_colocation_discovery + company: Unity Technologies + priority: 0 + required: 0 --- !u!114 &8410896261977003049 MonoBehaviour: m_ObjectHideFlags: 0 @@ -1565,7 +1692,8 @@ MonoBehaviour: nameUi: 'Meta Quest: Bounding Boxes' version: 0.1.0 featureIdInternal: com.unity.openxr.feature.arfoundation-meta-bounding-boxes - openxrExtensionStrings: XR_FB_scene XR_FB_spatial_entity XR_META_spatial_entity_discovery + openxrExtensionStrings: XR_FB_scene XR_FB_spatial_entity XR_FB_spatial_entity_container + XR_META_spatial_entity_discovery company: Unity Technologies priority: 0 required: 0 diff --git a/Assets/onnx/2camera_top20_45Smallmodel.onnx.meta b/Assets/onnx/2camera_top20_45Smallmodel.onnx.meta index d7b9d08..56d0df7 100644 --- a/Assets/onnx/2camera_top20_45Smallmodel.onnx.meta +++ b/Assets/onnx/2camera_top20_45Smallmodel.onnx.meta @@ -7,4 +7,7 @@ ScriptedImporter: userData: assetBundleName: assetBundleVariant: - script: {fileID: 11500000, guid: 683b6cb6d0a474744822c888b46772c9, type: 3} + script: {fileID: 11500000, guid: f22407ba6b4157b4a93d0a670bd3dd57, type: 3} + dynamicDimConfigs: + - name: batch_size + size: -1 diff --git a/Assets/onnx/baselineMask.onnx.meta b/Assets/onnx/baselineMask.onnx.meta index 503dc8a..9628599 100644 --- a/Assets/onnx/baselineMask.onnx.meta +++ b/Assets/onnx/baselineMask.onnx.meta @@ -7,4 +7,7 @@ ScriptedImporter: userData: assetBundleName: assetBundleVariant: - script: {fileID: 11500000, guid: 683b6cb6d0a474744822c888b46772c9, type: 3} + script: {fileID: 11500000, guid: f22407ba6b4157b4a93d0a670bd3dd57, type: 3} + dynamicDimConfigs: + - name: batch_size + size: -1 diff --git a/Assets/onnx/neuflow_mixed.onnx.meta b/Assets/onnx/neuflow_mixed.onnx.meta index ade6927..7c5eeaa 100644 --- a/Assets/onnx/neuflow_mixed.onnx.meta +++ b/Assets/onnx/neuflow_mixed.onnx.meta @@ -7,4 +7,5 @@ ScriptedImporter: userData: assetBundleName: assetBundleVariant: - script: {fileID: 11500000, guid: 683b6cb6d0a474744822c888b46772c9, type: 3} + script: {fileID: 11500000, guid: f22407ba6b4157b4a93d0a670bd3dd57, type: 3} + dynamicDimConfigs: [] diff --git a/Assets/onnx/train_multi_step2.onnx.meta b/Assets/onnx/train_multi_step2.onnx.meta index cdcae89..ab0f8d4 100644 --- a/Assets/onnx/train_multi_step2.onnx.meta +++ b/Assets/onnx/train_multi_step2.onnx.meta @@ -7,4 +7,7 @@ ScriptedImporter: userData: assetBundleName: assetBundleVariant: - script: {fileID: 11500000, guid: 683b6cb6d0a474744822c888b46772c9, type: 3} + script: {fileID: 11500000, guid: f22407ba6b4157b4a93d0a670bd3dd57, type: 3} + dynamicDimConfigs: + - name: batch_size + size: -1 diff --git a/Assets/onnx/yolov8n.onnx.meta b/Assets/onnx/yolov8n.onnx.meta index 0b14934..15ad25e 100644 --- a/Assets/onnx/yolov8n.onnx.meta +++ b/Assets/onnx/yolov8n.onnx.meta @@ -7,4 +7,5 @@ ScriptedImporter: userData: assetBundleName: assetBundleVariant: - script: {fileID: 11500000, guid: 683b6cb6d0a474744822c888b46772c9, type: 3} + script: {fileID: 11500000, guid: f22407ba6b4157b4a93d0a670bd3dd57, type: 3} + dynamicDimConfigs: [] diff --git a/Packages/manifest.json b/Packages/manifest.json index f95b828..d676a82 100644 --- a/Packages/manifest.json +++ b/Packages/manifest.json @@ -5,13 +5,13 @@ "com.unity.feature.development": "1.0.2", "com.unity.memoryprofiler": "1.1.6", "com.unity.multiplayer.center": "1.0.0", - "com.unity.sentis": "2.1.3", + "com.unity.sentis": "2.2.0", "com.unity.timeline": "1.8.7", "com.unity.ugui": "2.0.0", "com.unity.visualscripting": "1.9.8", "com.unity.xr.management": "4.5.1", - "com.unity.xr.meta-openxr": "2.1.1", - "com.unity.xr.openxr": "1.14.3", + "com.unity.xr.meta-openxr": "2.2.0", + "com.unity.xr.openxr": "1.15.1", "com.unity.modules.accessibility": "1.0.0", "com.unity.modules.ai": "1.0.0", "com.unity.modules.androidjni": "1.0.0", diff --git a/Packages/packages-lock.json b/Packages/packages-lock.json index 1007c99..7e87867 100644 --- a/Packages/packages-lock.json +++ b/Packages/packages-lock.json @@ -95,6 +95,17 @@ "dependencies": {}, "url": "https://packages.unity.com" }, + "com.unity.ai.inference": { + "version": "2.2.1", + "depth": 1, + "source": "registry", + "dependencies": { + "com.unity.burst": "1.8.17", + "com.unity.collections": "2.4.3", + "com.unity.modules.imageconversion": "1.0.0" + }, + "url": "https://packages.unity.com" + }, "com.unity.ai.navigation": { "version": "2.0.8", "depth": 2, @@ -105,7 +116,7 @@ "url": "https://packages.unity.com" }, "com.unity.burst": { - "version": "1.8.21", + "version": "1.8.23", "depth": 1, "source": "registry", "dependencies": { @@ -122,13 +133,14 @@ "url": "https://packages.unity.com" }, "com.unity.collections": { - "version": "2.5.1", + "version": "2.5.7", "depth": 1, "source": "registry", "dependencies": { - "com.unity.burst": "1.8.17", - "com.unity.test-framework": "1.4.5", - "com.unity.nuget.mono-cecil": "1.11.4", + "com.unity.burst": "1.8.19", + "com.unity.mathematics": "1.3.2", + "com.unity.test-framework": "1.4.6", + "com.unity.nuget.mono-cecil": "1.11.5", "com.unity.test-framework.performance": "3.0.3" }, "url": "https://packages.unity.com" @@ -178,7 +190,7 @@ "url": "https://packages.unity.com" }, "com.unity.inputsystem": { - "version": "1.14.0", + "version": "1.14.1", "depth": 1, "source": "registry", "dependencies": { @@ -215,7 +227,7 @@ } }, "com.unity.nuget.mono-cecil": { - "version": "1.11.4", + "version": "1.11.5", "depth": 2, "source": "registry", "dependencies": {}, @@ -243,15 +255,12 @@ "url": "https://packages.unity.com" }, "com.unity.sentis": { - "version": "2.1.3", + "version": "2.2.0", "depth": 0, - "source": "registry", + "source": "builtin", "dependencies": { - "com.unity.burst": "1.8.17", - "com.unity.collections": "2.4.3", - "com.unity.modules.imageconversion": "1.0.0" - }, - "url": "https://packages.unity.com" + "com.unity.ai.inference": "2.2.1" + } }, "com.unity.settings-manager": { "version": "2.1.0", @@ -330,7 +339,7 @@ "url": "https://packages.unity.com" }, "com.unity.xr.arfoundation": { - "version": "6.1.0", + "version": "6.2.0", "depth": 1, "source": "registry", "dependencies": { @@ -347,6 +356,17 @@ }, "url": "https://packages.unity.com" }, + "com.unity.xr.compositionlayers": { + "version": "2.1.0", + "depth": 1, + "source": "registry", + "dependencies": { + "com.unity.inputsystem": "1.6.3", + "com.unity.xr.core-utils": "2.2.2", + "com.unity.xr.management": "4.0.1" + }, + "url": "https://packages.unity.com" + }, "com.unity.xr.core-utils": { "version": "2.5.2", "depth": 1, @@ -380,19 +400,20 @@ "url": "https://packages.unity.com" }, "com.unity.xr.meta-openxr": { - "version": "2.1.1", + "version": "2.2.0", "depth": 0, "source": "registry", "dependencies": { - "com.unity.xr.openxr": "1.14.0", + "com.unity.xr.openxr": "1.15.0", "com.unity.xr.core-utils": "2.5.1", "com.unity.xr.management": "4.4.0", - "com.unity.xr.arfoundation": "6.1.0" + "com.unity.xr.arfoundation": "6.2.0", + "com.unity.xr.compositionlayers": "2.0.0" }, "url": "https://packages.unity.com" }, "com.unity.xr.openxr": { - "version": "1.14.3", + "version": "1.15.1", "depth": 0, "source": "registry", "dependencies": { @@ -551,7 +572,8 @@ "com.unity.modules.ui": "1.0.0", "com.unity.modules.imgui": "1.0.0", "com.unity.modules.jsonserialize": "1.0.0", - "com.unity.modules.hierarchycore": "1.0.0" + "com.unity.modules.hierarchycore": "1.0.0", + "com.unity.modules.physics": "1.0.0" } }, "com.unity.modules.umbra": { diff --git a/ProjectSettings/ProjectSettings.asset b/ProjectSettings/ProjectSettings.asset index 7e9d6f3..d85ab13 100644 --- a/ProjectSettings/ProjectSettings.asset +++ b/ProjectSettings/ProjectSettings.asset @@ -132,6 +132,7 @@ PlayerSettings: switchNVNMaxPublicSamplerIDCount: 0 switchMaxWorkerMultiple: 8 switchNVNGraphicsFirmwareMemory: 32 + switchGraphicsJobsSyncAfterKick: 1 vulkanNumSwapchainBuffers: 3 vulkanEnableSetSRGBWrite: 0 vulkanEnablePreTransform: 1 @@ -272,6 +273,9 @@ PlayerSettings: AndroidBuildApkPerCpuArchitecture: 0 AndroidTVCompatibility: 0 AndroidIsGame: 1 + androidAppCategory: 3 + useAndroidAppCategory: 1 + androidAppCategoryOther: AndroidEnableTango: 0 androidEnableBanner: 1 androidUseLowAccuracyLocation: 0 @@ -416,7 +420,7 @@ PlayerSettings: - m_BuildTarget: iOSSupport m_GraphicsJobs: 0 - m_BuildTarget: WindowsStandaloneSupport - m_GraphicsJobs: 0 + m_GraphicsJobs: 1 - m_BuildTarget: XboxOnePlayer m_GraphicsJobs: 1 - m_BuildTarget: LuminSupport @@ -430,6 +434,8 @@ PlayerSettings: m_GraphicsJobMode: 0 - m_BuildTarget: XboxOnePlayer m_GraphicsJobMode: 0 + - m_BuildTarget: WindowsStandaloneSupport + m_GraphicsJobMode: 1 m_BuildTargetGraphicsAPIs: - m_BuildTarget: AndroidPlayer m_APIs: 15000000 @@ -761,8 +767,9 @@ PlayerSettings: webWasm2023: 0 webEnableSubmoduleStrippingCompatibility: 0 scriptingDefineSymbols: - Android: ISDK_OPENXR_HAND - Standalone: ISDK_OPENXR_HAND + Android: ISDK_OPENXR_HAND;USE_INPUT_SYSTEM_POSE_CONTROL;USE_STICK_CONTROL_THUMBSTICKS + Standalone: ISDK_OPENXR_HAND;USE_INPUT_SYSTEM_POSE_CONTROL;USE_STICK_CONTROL_THUMBSTICKS + Windows Store Apps: USE_INPUT_SYSTEM_POSE_CONTROL;USE_STICK_CONTROL_THUMBSTICKS additionalCompilerArguments: {} platformArchitecture: {} scriptingBackend: diff --git a/ProjectSettings/ProjectVersion.txt b/ProjectSettings/ProjectVersion.txt index d581360..004a7db 100644 --- a/ProjectSettings/ProjectVersion.txt +++ b/ProjectSettings/ProjectVersion.txt @@ -1,2 +1,2 @@ -m_EditorVersion: 6000.1.7f1 -m_EditorVersionWithRevision: 6000.1.7f1 (13a8ffad9172) +m_EditorVersion: 6000.2.0f1 +m_EditorVersionWithRevision: 6000.2.0f1 (eed1c594c913) From 2ce902c4a340e3ad8457a18836ea089a1cf8040c Mon Sep 17 00:00:00 2001 From: Faisal Zaghloul Date: Tue, 19 Aug 2025 13:41:21 -0400 Subject: [PATCH 2/8] Pulling texture data from SpotObserver + prompda model pushed to lfs --- Assets/DepthManager.cs | 3 +- Assets/Scenes/ghost.unity | 50 ++++++++++++++++++++- Assets/Scripts/DrawMeshInstanced.cs | 39 +++++++++-------- Assets/Scripts/SpotObserverClient.cs | 65 ++++++++++++++++++++-------- 4 files changed, 119 insertions(+), 38 deletions(-) diff --git a/Assets/DepthManager.cs b/Assets/DepthManager.cs index 8205d0d..7f562ba 100644 --- a/Assets/DepthManager.cs +++ b/Assets/DepthManager.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using UnityEngine; using UnityEngine.UI; +using Unity.InferenceEngine; using Unity.VisualScripting; @@ -209,7 +210,7 @@ private IEnumerator ResetActivateDepthEstimation() public (ComputeBuffer, Matrix4x4, float[]) update_depth_from_renderer(Texture2D rgb, float[] depth, int camera_index, bool calculate_icp, bool new_depth, bool avg_before_complete) { - Unity.InferenceEngine.TextureTransform tform = new(); + TextureTransform tform = new(); tform.SetDimensions(rgb.width, rgb.height, 3); diff --git a/Assets/Scenes/ghost.unity b/Assets/Scenes/ghost.unity index 8840451..7e00760 100644 --- a/Assets/Scenes/ghost.unity +++ b/Assets/Scenes/ghost.unity @@ -4196,6 +4196,10 @@ PrefabInstance: propertyPath: Mass value: 1.904699 objectReference: {fileID: 0} + - target: {fileID: 1137478775010377568, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: width + value: 640 + objectReference: {fileID: 0} - target: {fileID: 1137478775010377568, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: y_min value: 0.5 @@ -4204,6 +4208,10 @@ PrefabInstance: propertyPath: z_max value: 100 objectReference: {fileID: 0} + - target: {fileID: 1137478775010377568, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: height + value: 480 + objectReference: {fileID: 0} - target: {fileID: 1137478775010377568, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: target value: @@ -4236,6 +4244,14 @@ PrefabInstance: propertyPath: savePointCloud value: 0 objectReference: {fileID: 0} + - target: {fileID: 1137478775010377568, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: colorSubscriber + value: + objectReference: {fileID: 0} + - target: {fileID: 1137478775010377568, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: depthSubscriber + value: + objectReference: {fileID: 0} - target: {fileID: 1137478775010377568, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: use_saved_meshes value: 1 @@ -4584,6 +4600,10 @@ PrefabInstance: propertyPath: use_saved_meshes value: 1 objectReference: {fileID: 0} + - target: {fileID: 3310871052733856933, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: SpotObserverCameraIndex + value: 4 + objectReference: {fileID: 0} - target: {fileID: 3310871052733856935, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: m_IsActive value: 1 @@ -4696,6 +4716,10 @@ PrefabInstance: propertyPath: use_saved_meshes value: 1 objectReference: {fileID: 0} + - target: {fileID: 3310871053306987607, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: SpotObserverCameraIndex + value: 2 + objectReference: {fileID: 0} - target: {fileID: 3310871053646093096, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: Topic value: /spot/stream_image2/frontleft_fisheye_image/image @@ -4824,6 +4848,22 @@ PrefabInstance: propertyPath: rightPress value: 4 objectReference: {fileID: 0} + - target: {fileID: 4598551965991344262, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: 'colorSubscribers.Array.data[0]' + value: + objectReference: {fileID: 0} + - target: {fileID: 4598551965991344262, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: 'colorSubscribers.Array.data[1]' + value: + objectReference: {fileID: 0} + - target: {fileID: 4598551965991344262, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: 'depthSubscribers.Array.data[0]' + value: + objectReference: {fileID: 0} + - target: {fileID: 4598551965991344262, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: 'depthSubscribers.Array.data[1]' + value: + objectReference: {fileID: 0} - target: {fileID: 4598551965991344281, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: objectsOfModeToEnable.Array.size value: 0 @@ -4940,6 +4980,14 @@ PrefabInstance: propertyPath: rightController value: objectReference: {fileID: 1052391961} + - target: {fileID: 5143370623281230805, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: 'depthSubscribers.Array.data[0]' + value: + objectReference: {fileID: 0} + - target: {fileID: 5143370623281230805, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: 'depthSubscribers.Array.data[1]' + value: + objectReference: {fileID: 0} - target: {fileID: 6095128873530598835, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: Mass value: 1.904699 @@ -9955,7 +10003,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: cdc798fc4b250c04b91b6a0767d25d31, type: 3} m_Name: m_EditorClassIdentifier: - RobotIP: 128.148.140.22 + RobotIP: 128.148.138.88 username: user password: bigbubbabigbubba EnableDebugDumps: 0 diff --git a/Assets/Scripts/DrawMeshInstanced.cs b/Assets/Scripts/DrawMeshInstanced.cs index 4dc0432..d78d0fa 100644 --- a/Assets/Scripts/DrawMeshInstanced.cs +++ b/Assets/Scripts/DrawMeshInstanced.cs @@ -1,17 +1,19 @@ -using System.Collections; -using System.Collections.Generic; -using UnityEngine; -using System.IO; +using JetBrains.Annotations; using RosSharp.RosBridgeClient; using RosSharp.RosBridgeClient.MessageTypes.Nav; -using System.Text; -using static RosSharp.Urdf.Link.Visual.Material; using System; -using JetBrains.Annotations; +using System.Collections; +using System.Collections.Generic; using System.Diagnostics; -using Debug = UnityEngine.Debug; +using System.IO; +using System.Text; +using Unity.InferenceEngine; using Unity.Mathematics; using Unity.VisualScripting; +using UnityEngine; +using static RosSharp.Urdf.Link.Visual.Material; +using static SpotObserverClient; +using Debug = UnityEngine.Debug; //using System; @@ -27,12 +29,15 @@ public class DrawMeshInstanced : MonoBehaviour public DepthManager depthManager; public int camera_index; + public SpotCamera SpotObserverCameraIndex; + public Transform mainCameraRot; public float range; public Texture2D color_image; public Texture2D depth_image; + public Tensor depth_tensor; public int imageScriptIndex; @@ -400,24 +405,24 @@ private void UpdateTexture() Debug.Log("Using saved mesh"); for (int i = 0; i < 480 * 640; i++) { - //depth_ar[i] = depth_ar_saved[i]; - depth_ar[i] = 1.0f; + depth_ar[i] = depth_ar_saved[i]; } - (color_image, depth_image) = spotObserverClient.GetCameraFeeds(0); - Debug.Log("Color image: " + (color_image == null ? "null" : color_image.width + "x" + color_image.height)); + (color_image, depth_tensor) = spotObserverClient.GetCameraFeeds(0); - //if (depth_image != null) - //{ - // depth_ar = depth_image.GetPixelData(0).ToArray(); - //} + Debug.Log("Color image: " + (color_image == null ? "null" : color_image.width + "x" + color_image.height)); + // HACK + if (depth_tensor != null) + { + ComputeTensorData.Pin(depth_tensor).buffer.GetData(depth_ar); + } new_depth_to_render = true; } else { Debug.Log("Getting camera feeds from Spot Observer Client"); - (color_image, depth_image) = spotObserverClient.GetCameraFeeds(0); + (color_image, depth_tensor) = spotObserverClient.GetCameraFeeds(0); Debug.Log("Color image: " + (color_image == null ? "null" : color_image.width + "x" + color_image.height)); if (depth_image != null) diff --git a/Assets/Scripts/SpotObserverClient.cs b/Assets/Scripts/SpotObserverClient.cs index 8c691b5..c7449dc 100644 --- a/Assets/Scripts/SpotObserverClient.cs +++ b/Assets/Scripts/SpotObserverClient.cs @@ -1,6 +1,7 @@ using RosSharp.RosBridgeClient.MessageTypes.Std; using System; using System.Runtime.InteropServices; +using Unity.Collections; using Unity.InferenceEngine; using UnityEngine; @@ -82,11 +83,12 @@ private static void PluginLogCallback(string message) private Texture2D[] depth_textures; //private Tensor[] rgb_tensors; - //private Tensor[] depth_tensors; + private Tensor[] depth_tensors; private IntPtr[] rgb_resources; private IntPtr[] depth_resources; + private NativeHashMap SpotCamToIdx; void Start() { @@ -124,33 +126,38 @@ void Start() // new Tensor(rgb_shape), // new Tensor(rgb_shape) //}; - //depth_tensors = new Tensor[] - //{ - // new Tensor(depth_shape), - // new Tensor(depth_shape), - // new Tensor(depth_shape) - //}; + depth_tensors = new Tensor[] + { + new Tensor(depth_shape), + new Tensor(depth_shape), + new Tensor(depth_shape) + }; uint[] cams = { - (uint)SpotCamera.FRONTRIGHT, (uint)SpotCamera.FRONTLEFT, + (uint)SpotCamera.FRONTRIGHT, (uint)SpotCamera.HAND, - }; + SpotCamToIdx = new NativeHashMap(cams.Length, Allocator.Persistent); + for (int i = 0; i < cams.Length; i++) + { + SpotCamToIdx[(int)cams[i]] = i; + } + uint all_cams = 0; for (var i = 0; i < rgb_textures.Length; i++) { rgb_textures[i] = new Texture2D(640, 480, TextureFormat.RGB24, false); - depth_textures[i] = new Texture2D(640, 480, TextureFormat.RFloat, false); + //depth_textures[i] = new Texture2D(640, 480, TextureFormat.RFloat, false); rgb_resources[i] = rgb_textures[i].GetNativeTexturePtr(); - depth_resources[i] = depth_textures[i].GetNativeTexturePtr(); + //depth_resources[i] = depth_textures[i].GetNativeTexturePtr(); //rgb_resources[i] = ComputeTensorData.Pin(rgb_tensors[i]).buffer.GetNativeBufferPtr(); - //depth_resources[i] = ComputeTensorData.Pin(depth_tensors[i]).buffer.GetNativeBufferPtr(); + depth_resources[i] = ComputeTensorData.Pin(depth_tensors[i]).buffer.GetNativeBufferPtr(); // Register textures with the Spot observer - if (!SOb_RegisterUnityReadbackBuffers(robot_id, cams[i], rgb_resources[i], depth_resources[i], 640 * 480 * 3, 640 * 480 * 4)) + if (!SOb_RegisterUnityReadbackBuffers(robot_id, cams[i], rgb_resources[i], depth_resources[i], 640 * 480 * 4, 640 * 480 * 4)) { Debug.LogError("Failed to register textures for camera " + cams[i]); } @@ -186,10 +193,10 @@ void OnDestroy() //{ // tensor.Dispose(); //} - //foreach (var tensor in depth_tensors) - //{ - // tensor.Dispose(); - //} + foreach (var tensor in depth_tensors) + { + tensor.Dispose(); + } foreach (var texture in rgb_textures) { if (texture != null) @@ -197,6 +204,14 @@ void OnDestroy() Destroy(texture); } } + foreach (var texture in depth_textures) + { + if (texture != null) + { + Destroy(texture); + } + } + SpotCamToIdx.Dispose(); } void Update() @@ -215,7 +230,7 @@ void Update() Debug.LogWarning("SUCCESSFULLY READ CAMERA FEEDS"); } - public (Texture2D, Texture2D) GetCameraFeeds(int id) + public (Texture2D, Tensor) GetCameraFeeds(SpotCamera id) { if (!isConnected) { @@ -223,6 +238,18 @@ void Update() return (null, null); } Debug.LogWarning("Returning RGB and Depth for " + id); - return (rgb_textures[id], depth_textures[id]); + + for (int i = 0; i < 3; i++) + { + Debug.Log("Camera " + SpotCamToIdx[i] + " mapped to index " + SpotCamToIdx[(int)SpotCamToIdx[i]]); + } + + + if (!SpotCamToIdx.TryGetValue((int)id, out int idx)) + { + Debug.LogError("Invalid camera ID: " + id); + return (null, null); + } + return (rgb_textures[idx], depth_tensors[idx]); } } From 68b11a1e85073a19bf49f49aa1fde64d94d59db9 Mon Sep 17 00:00:00 2001 From: Faisal Zaghloul Date: Tue, 19 Aug 2025 13:50:11 -0400 Subject: [PATCH 3/8] Spot observer works! --- .gitattributes | 1 + Assets/Scenes/ghost.unity | 8 +++++ Assets/Scripts/DrawMeshInstanced.cs | 8 +++-- Assets/Scripts/SpotObserverClient.cs | 45 +++++++++++----------------- 4 files changed, 32 insertions(+), 30 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..12250c6 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +Assets/onnx/promptda/prompt-depth-anything-vitl-rotated_optimized.onnx filter=lfs diff=lfs merge=lfs -text diff --git a/Assets/Scenes/ghost.unity b/Assets/Scenes/ghost.unity index 7e00760..043576c 100644 --- a/Assets/Scenes/ghost.unity +++ b/Assets/Scenes/ghost.unity @@ -4260,6 +4260,10 @@ PrefabInstance: propertyPath: spotObserverClient value: objectReference: {fileID: 971801071} + - target: {fileID: 1137478775010377568, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: SpotObserverCameraIndex + value: 2 + objectReference: {fileID: 0} - target: {fileID: 1137478775010377574, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: m_IsActive value: 1 @@ -4360,6 +4364,10 @@ PrefabInstance: propertyPath: spotObserverClient value: objectReference: {fileID: 971801071} + - target: {fileID: 1137478775515482002, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: SpotObserverCameraIndex + value: 4 + objectReference: {fileID: 0} - target: {fileID: 1137478775531494915, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: RosBridgeServerUrl value: ws://128.148.138.132:9090 diff --git a/Assets/Scripts/DrawMeshInstanced.cs b/Assets/Scripts/DrawMeshInstanced.cs index d78d0fa..a736d2d 100644 --- a/Assets/Scripts/DrawMeshInstanced.cs +++ b/Assets/Scripts/DrawMeshInstanced.cs @@ -405,10 +405,12 @@ private void UpdateTexture() Debug.Log("Using saved mesh"); for (int i = 0; i < 480 * 640; i++) { - depth_ar[i] = depth_ar_saved[i]; + //depth_ar[i] = depth_ar_saved[i]; + depth_ar[i] = 2.0f; } - (color_image, depth_tensor) = spotObserverClient.GetCameraFeeds(0); + Debug.Log("SpotObserverCameraIndex: " + (int)SpotObserverCameraIndex); + (color_image, depth_tensor) = spotObserverClient.GetCameraFeeds(SpotObserverCameraIndex); Debug.Log("Color image: " + (color_image == null ? "null" : color_image.width + "x" + color_image.height)); // HACK @@ -422,7 +424,7 @@ private void UpdateTexture() else { Debug.Log("Getting camera feeds from Spot Observer Client"); - (color_image, depth_tensor) = spotObserverClient.GetCameraFeeds(0); + //(color_image, depth_tensor) = spotObserverClient.GetCameraFeeds(0); Debug.Log("Color image: " + (color_image == null ? "null" : color_image.width + "x" + color_image.height)); if (depth_image != null) diff --git a/Assets/Scripts/SpotObserverClient.cs b/Assets/Scripts/SpotObserverClient.cs index c7449dc..b684bbf 100644 --- a/Assets/Scripts/SpotObserverClient.cs +++ b/Assets/Scripts/SpotObserverClient.cs @@ -90,6 +90,11 @@ private static void PluginLogCallback(string message) private NativeHashMap SpotCamToIdx; + private uint[] cams = { + (uint)SpotCamera.FRONTLEFT, + (uint)SpotCamera.FRONTRIGHT + }; + void Start() { SOb_SetUnityLogCallback(PluginLogCallback); @@ -114,11 +119,13 @@ void Start() TensorShape rgb_shape = new TensorShape(1, 3, 480, 640); // RGB shape TensorShape depth_shape = new TensorShape(1, 1, 480, 640); // Depth shape + int num_cams = cams.Length; + // Initialize textures and tensors for each camera - rgb_textures = new Texture2D[3]; - depth_textures = new Texture2D[3]; - rgb_resources = new IntPtr[3]; - depth_resources = new IntPtr[3]; + rgb_textures = new Texture2D[num_cams]; + depth_textures = new Texture2D[num_cams]; + rgb_resources = new IntPtr[num_cams]; + depth_resources = new IntPtr[num_cams]; //rgb_tensors = new Tensor[] //{ @@ -126,18 +133,7 @@ void Start() // new Tensor(rgb_shape), // new Tensor(rgb_shape) //}; - depth_tensors = new Tensor[] - { - new Tensor(depth_shape), - new Tensor(depth_shape), - new Tensor(depth_shape) - }; - - uint[] cams = { - (uint)SpotCamera.FRONTLEFT, - (uint)SpotCamera.FRONTRIGHT, - (uint)SpotCamera.HAND, - }; + depth_tensors = new Tensor[num_cams]; SpotCamToIdx = new NativeHashMap(cams.Length, Allocator.Persistent); for (int i = 0; i < cams.Length; i++) @@ -146,7 +142,7 @@ void Start() } uint all_cams = 0; - for (var i = 0; i < rgb_textures.Length; i++) + for (var i = 0; i < num_cams; i++) { rgb_textures[i] = new Texture2D(640, 480, TextureFormat.RGB24, false); //depth_textures[i] = new Texture2D(640, 480, TextureFormat.RFloat, false); @@ -154,6 +150,7 @@ void Start() //depth_resources[i] = depth_textures[i].GetNativeTexturePtr(); //rgb_resources[i] = ComputeTensorData.Pin(rgb_tensors[i]).buffer.GetNativeBufferPtr(); + depth_tensors[i] = new Tensor(depth_shape); depth_resources[i] = ComputeTensorData.Pin(depth_tensors[i]).buffer.GetNativeBufferPtr(); // Register textures with the Spot observer @@ -211,7 +208,7 @@ void OnDestroy() Destroy(texture); } } - SpotCamToIdx.Dispose(); + SpotCamToIdx.Clear(); } void Update() @@ -237,19 +234,13 @@ void Update() Debug.LogError("Not connected to Spot robot. Cannot get camera feeds."); return (null, null); } - Debug.LogWarning("Returning RGB and Depth for " + id); - - for (int i = 0; i < 3; i++) - { - Debug.Log("Camera " + SpotCamToIdx[i] + " mapped to index " + SpotCamToIdx[(int)SpotCamToIdx[i]]); - } - - if (!SpotCamToIdx.TryGetValue((int)id, out int idx)) { - Debug.LogError("Invalid camera ID: " + id); + Debug.LogError("Invalid camera ID: " + (int)id); return (null, null); } + Debug.Log("Returning RGB and Depth for " + id); + return (rgb_textures[idx], depth_tensors[idx]); } } From a26274de946b5722fb61c91d040dd4e14ab09e9d Mon Sep 17 00:00:00 2001 From: Faisal Zaghloul Date: Wed, 20 Aug 2025 12:13:39 -0400 Subject: [PATCH 4/8] Enable 2-spot system using SpotObserver --- Assets/Scenes/ghost.unity | 68 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/Assets/Scenes/ghost.unity b/Assets/Scenes/ghost.unity index 043576c..6763ea8 100644 --- a/Assets/Scenes/ghost.unity +++ b/Assets/Scenes/ghost.unity @@ -4608,6 +4608,10 @@ PrefabInstance: propertyPath: use_saved_meshes value: 1 objectReference: {fileID: 0} + - target: {fileID: 3310871052733856933, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: spotObserverClient + value: + objectReference: {fileID: 1028337600} - target: {fileID: 3310871052733856933, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: SpotObserverCameraIndex value: 4 @@ -4724,6 +4728,10 @@ PrefabInstance: propertyPath: use_saved_meshes value: 1 objectReference: {fileID: 0} + - target: {fileID: 3310871053306987607, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: spotObserverClient + value: + objectReference: {fileID: 1028337600} - target: {fileID: 3310871053306987607, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: SpotObserverCameraIndex value: 2 @@ -5054,6 +5062,9 @@ PrefabInstance: - targetCorrespondingSourceObject: {fileID: 3310871052953474555, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} insertIndex: 0 addedObject: {fileID: 1834046043} + - targetCorrespondingSourceObject: {fileID: 3310871053554185043, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + insertIndex: -1 + addedObject: {fileID: 1028337601} - targetCorrespondingSourceObject: {fileID: 3310871052371914859, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} insertIndex: 0 addedObject: {fileID: 1444031539} @@ -5522,6 +5533,11 @@ Transform: m_Children: [] m_Father: {fileID: 360447432} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!4 &475214583 stripped +Transform: + m_CorrespondingSourceObject: {fileID: 3310871053554185043, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + m_PrefabInstance: {fileID: 409472599} + m_PrefabAsset: {fileID: 0} --- !u!1 &477272451 GameObject: m_ObjectHideFlags: 0 @@ -10337,6 +10353,54 @@ Transform: m_Children: [] m_Father: {fileID: 1397146816} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!1 &1028337599 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 1028337601} + - component: {fileID: 1028337600} + m_Layer: 0 + m_Name: SpotObserverClient + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!114 &1028337600 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1028337599} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: cdc798fc4b250c04b91b6a0767d25d31, type: 3} + m_Name: + m_EditorClassIdentifier: + RobotIP: 128.148.138.87 + username: user + password: bigbubbabigbubba + EnableDebugDumps: 0 +--- !u!4 &1028337601 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1028337599} + serializedVersion: 2 + m_LocalRotation: {x: -0, y: -0, z: -0, w: 1} + m_LocalPosition: {x: 0.060298983, y: 0.042863965, z: -0.43645263} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 475214583} + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} --- !u!1 &1028943332 GameObject: m_ObjectHideFlags: 0 @@ -14861,11 +14925,11 @@ MonoBehaviour: meanThreshold: 0.1 activate_CVD: 0 cvd_weight: 3 - show_spot: 0 + show_spot: 1 another_manager: {fileID: 638278051} CVDLeft: {fileID: 1031760795} CVDRight: {fileID: 1274553787} - activate_depth_estimation: 1 + activate_depth_estimation: 0 activate_ICP: 0 activate_edge_detection: 1 edgeThreshold: 21 From c4971a3154f828ca16c8881bf972713ed32cb204 Mon Sep 17 00:00:00 2001 From: Faisal Zaghloul Date: Wed, 20 Aug 2025 22:16:13 -0400 Subject: [PATCH 5/8] add promptda to lfs --- Assets/Scenes/ghost.unity | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Assets/Scenes/ghost.unity b/Assets/Scenes/ghost.unity index 6763ea8..0a9bed7 100644 --- a/Assets/Scenes/ghost.unity +++ b/Assets/Scenes/ghost.unity @@ -5185,7 +5185,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 42c5c42437e094441929da3e64cb7559, type: 3} m_Name: m_EditorClassIdentifier: - UB_Model_file: C:/Users/brown/Documents/fmz/GHOST/Assets/onnx/promptda/prompt-depth-anything-vitl-rotated_optimized.onnx + UB_Model_file: C:/Users/brown/Documents/fmz/GHOST/Assets/onnxqg-vitl-rotated_optimized.onnx UB_debug_dump: 1 --- !u!4 &420483825 Transform: @@ -10027,7 +10027,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: cdc798fc4b250c04b91b6a0767d25d31, type: 3} m_Name: m_EditorClassIdentifier: - RobotIP: 128.148.138.88 + RobotIP: 128.148.138.21 username: user password: bigbubbabigbubba EnableDebugDumps: 0 From 0705ba56c90fc8ccf790ed55dcbfe314dd32739b Mon Sep 17 00:00:00 2001 From: Faisal Zaghloul Date: Sat, 23 Aug 2025 11:26:16 -0400 Subject: [PATCH 6/8] Hooking in completed vision pipeline --- Assets/Scenes/ghost.unity | 16 +++++---- Assets/Scripts/SpotObserverClient.cs | 50 +++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/Assets/Scenes/ghost.unity b/Assets/Scenes/ghost.unity index 0a9bed7..6378977 100644 --- a/Assets/Scenes/ghost.unity +++ b/Assets/Scenes/ghost.unity @@ -5185,7 +5185,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 42c5c42437e094441929da3e64cb7559, type: 3} m_Name: m_EditorClassIdentifier: - UB_Model_file: C:/Users/brown/Documents/fmz/GHOST/Assets/onnxqg-vitl-rotated_optimized.onnx + UB_Model_file: C:/Users/brown/Documents/fmz/GHOST/Assets/onnx/promptda/prompt-depth-anything-vitl-rotated_optimized.onnx UB_debug_dump: 1 --- !u!4 &420483825 Transform: @@ -7095,7 +7095,7 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: rgbdModelAsset: {fileID: 5022602860645237092, guid: 5c0adad9409f35a498f591972a8c1bec, type: 3} - Use_UB: 0 + Use_UB: 1 UB_Model: {fileID: 420483824} --- !u!114 &638278051 MonoBehaviour: @@ -7118,7 +7118,7 @@ MonoBehaviour: another_manager: {fileID: 1639569377} CVDLeft: {fileID: 305849529} CVDRight: {fileID: 1814602523} - activate_depth_estimation: 0 + activate_depth_estimation: 1 activate_ICP: 0 activate_edge_detection: 1 edgeThreshold: 21 @@ -10027,10 +10027,11 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: cdc798fc4b250c04b91b6a0767d25d31, type: 3} m_Name: m_EditorClassIdentifier: - RobotIP: 128.148.138.21 + RobotIP: 128.148.138.88 username: user password: bigbubbabigbubba - EnableDebugDumps: 0 + useVisionPipeline: 1 + enableDebugDumps: 0 --- !u!1 &973382266 GameObject: m_ObjectHideFlags: 0 @@ -10385,7 +10386,8 @@ MonoBehaviour: RobotIP: 128.148.138.87 username: user password: bigbubbabigbubba - EnableDebugDumps: 0 + useVisionPipeline: 1 + enableDebugDumps: 0 --- !u!4 &1028337601 Transform: m_ObjectHideFlags: 0 @@ -14925,7 +14927,7 @@ MonoBehaviour: meanThreshold: 0.1 activate_CVD: 0 cvd_weight: 3 - show_spot: 1 + show_spot: 0 another_manager: {fileID: 638278051} CVDLeft: {fileID: 1031760795} CVDRight: {fileID: 1274553787} diff --git a/Assets/Scripts/SpotObserverClient.cs b/Assets/Scripts/SpotObserverClient.cs index b684bbf..456a450 100644 --- a/Assets/Scripts/SpotObserverClient.cs +++ b/Assets/Scripts/SpotObserverClient.cs @@ -41,6 +41,23 @@ private static extern bool SOb_PushNextImageSetToUnityBuffers( int robot_id ); + [DllImport("SpotObserverLib", CharSet = CharSet.Ansi)] + private static extern IntPtr SOb_LoadModel(string modelPath, string backend); + + [DllImport("SpotObserverLib")] + private static extern void SOb_UnloadModel(IntPtr model); + + [DllImport("SpotObserverLib")] + private static extern bool SOb_LaunchVisionPipeline(int robot_id, IntPtr model); + + [DllImport("SpotObserverLib")] + private static extern bool SOb_StopVisionPipeline(int robot_id); + + [DllImport("SpotObserverLib")] + private static extern bool SOb_PushNextVisionPipelineImageSetToUnityBuffers( + int robot_id + ); + [DllImport("SpotObserverLib")] private static extern bool SOb_SetUnityLogCallback( LogCallback callback @@ -51,6 +68,7 @@ private static extern void SOb_ToggleDebugDumps( string dump_path ); + // Make sure the enum matches the C++ enum in spot-observer.h public enum SpotCamera { @@ -74,10 +92,12 @@ private static void PluginLogCallback(string message) public string username; public string password; - public bool EnableDebugDumps = false; + public bool useVisionPipeline = true; + public bool enableDebugDumps = false; private int robot_id = -1; private bool isConnected = false; + private IntPtr model; private Texture2D[] rgb_textures; private Texture2D[] depth_textures; @@ -94,7 +114,7 @@ private static void PluginLogCallback(string message) (uint)SpotCamera.FRONTLEFT, (uint)SpotCamera.FRONTRIGHT }; - + void Start() { SOb_SetUnityLogCallback(PluginLogCallback); @@ -109,7 +129,23 @@ void Start() Debug.Log("Connected to Spot robot at " + RobotIP); isConnected = true; - if (EnableDebugDumps) + model = SOb_LoadModel("vision_pipeline.onnx", "cuda"); + if (model == IntPtr.Zero) + { + Debug.LogError("Failed to load vision pipeline model."); + isConnected = false; + return; + } + + bool ret = SOb_LaunchVisionPipeline(robot_id, model); + if (!ret) + { + Debug.LogError("Failed to launch vision pipeline on Spot robot."); + isConnected = false; + return; + } + + if (enableDebugDumps) { SOb_ToggleDebugDumps("SPOT_OBSERVER_DUMPS"); } @@ -219,11 +255,17 @@ void Update() return; } - if (!SOb_PushNextImageSetToUnityBuffers(robot_id)) + //if (!SOb_PushNextImageSetToUnityBuffers(robot_id)) + //{ + // return; + //} + + if (!SOb_PushNextVisionPipelineImageSetToUnityBuffers(robot_id)) { return; } + Debug.LogWarning("SUCCESSFULLY READ CAMERA FEEDS"); } From 20fa8fb60ac3bf1c8aed83870cdaf8e43b65df4b Mon Sep 17 00:00:00 2001 From: Faisal Zaghloul Date: Mon, 25 Aug 2025 15:54:53 -0400 Subject: [PATCH 7/8] first version of new vision pipeline works --- Assets/DepthManager.cs | 4 +- .../2poseCVDShaderLeft.compute.meta | 7 - .../PointClouds/2poseCVDShaderRight.compute | 282 ------------------ .../2poseCVDShaderRight.compute.meta | 7 - .../PointClouds/InstancedIndirectColor.shader | 3 +- ...aderLeft.compute => poseCVDShader.compute} | 99 ------ ...ompute.meta => poseCVDShader.compute.meta} | 0 Assets/PointClouds/poseCVDShaderLeft.compute | 282 ------------------ Assets/PointClouds/poseCVDShaderRight.compute | 282 ------------------ .../poseCVDShaderRight.compute.meta | 7 - Assets/PoseConsistentVideoDepth.cs | 52 ++-- Assets/Scenes/ghost.unity | 266 ++++++++++++++++- Assets/Scripts/DrawMeshInstanced.cs | 47 ++- Assets/Scripts/SpotObserverClient.cs | 146 +++++---- 14 files changed, 411 insertions(+), 1073 deletions(-) delete mode 100644 Assets/PointClouds/2poseCVDShaderLeft.compute.meta delete mode 100644 Assets/PointClouds/2poseCVDShaderRight.compute delete mode 100644 Assets/PointClouds/2poseCVDShaderRight.compute.meta rename Assets/PointClouds/{2poseCVDShaderLeft.compute => poseCVDShader.compute} (62%) rename Assets/PointClouds/{poseCVDShaderLeft.compute.meta => poseCVDShader.compute.meta} (100%) delete mode 100644 Assets/PointClouds/poseCVDShaderLeft.compute delete mode 100644 Assets/PointClouds/poseCVDShaderRight.compute delete mode 100644 Assets/PointClouds/poseCVDShaderRight.compute.meta diff --git a/Assets/DepthManager.cs b/Assets/DepthManager.cs index 7f562ba..3620b22 100644 --- a/Assets/DepthManager.cs +++ b/Assets/DepthManager.cs @@ -307,8 +307,8 @@ private IEnumerator ResetActivateDepthEstimation() AveragerRight.update_depth_buffer(temp_depth_right); - temp_depth_left_return = CVDLeft.consistent_depth(temp_depth_left, mat_l, temp_optical_left, activate_CVD && is_not_moving, edgethreshold, activate_edge_detection, activate_depth_estimation, cvd_weight); - temp_depth_right_return = CVDRight.consistent_depth(temp_depth_right, mat_r, temp_optical_right, activate_CVD && is_not_moving, edgethreshold, activate_edge_detection, activate_depth_estimation, cvd_weight); + temp_depth_left_return = CVDLeft.consistent_depth(temp_depth_left, mat_l, temp_optical_left, activate_CVD && is_not_moving, edgethreshold, activate_depth_estimation, cvd_weight); + temp_depth_right_return = CVDRight.consistent_depth(temp_depth_right, mat_r, temp_optical_right, activate_CVD && is_not_moving, edgethreshold, activate_depth_estimation, cvd_weight); return (temp_depth_left_return, temp_depth_right_return, ICP_trans); } diff --git a/Assets/PointClouds/2poseCVDShaderLeft.compute.meta b/Assets/PointClouds/2poseCVDShaderLeft.compute.meta deleted file mode 100644 index 7d2a95e..0000000 --- a/Assets/PointClouds/2poseCVDShaderLeft.compute.meta +++ /dev/null @@ -1,7 +0,0 @@ -fileFormatVersion: 2 -guid: 77636f91f1ca1a642968ac70d7055d9f -ComputeShaderImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Assets/PointClouds/2poseCVDShaderRight.compute b/Assets/PointClouds/2poseCVDShaderRight.compute deleted file mode 100644 index e13e717..0000000 --- a/Assets/PointClouds/2poseCVDShaderRight.compute +++ /dev/null @@ -1,282 +0,0 @@ -// Each #kernel tells which function to compile; you can have many kernels -#pragma kernel Transformation -#pragma kernel EdgeDetection -#pragma kernel CVD - -#define WIDTH 640 -#define HEIGHT 480 - -// Number of frames -int num_frames; - -// Buffers -RWStructuredBuffer depth_ar; -RWStructuredBuffer optical_ar; -RWStructuredBuffer output_ar; -RWStructuredBuffer depth_buffer; -RWStructuredBuffer optical_buffer; -RWStructuredBuffer pose_buffer; - -// pose mat -float4x4 pose; -float4x4 inverse_pose; - -// buffer pos -int buffer_pos; - -// intrinsics -float4 intrinsics; - -// edge detection -float edgethreshold; - -// weight -float cvd_weight; - -// EdgeDetection -[numthreads(16, 8, 1)] -void Transformation(uint3 groupId : SV_GroupID, uint3 threadId : SV_GroupThreadID) -{ - int xIndex = groupId.x * 16 + threadId.x; - int yIndex = groupId.y * 8 + threadId.y; - - int index = xIndex + yIndex * WIDTH; - - if (xIndex > 0 && xIndex < WIDTH - 1 && yIndex > 0 && yIndex < HEIGHT - 1) - { - uint depth_idx = (WIDTH * (HEIGHT - yIndex - 1)) + (WIDTH - xIndex - 1); - - float3 pos; - pos.z = depth_ar[index]; - pos.x = (xIndex - intrinsics.x) * pos.z / intrinsics.z; - pos.y = (yIndex - intrinsics.y) * pos.z / intrinsics.w; - - depth_buffer[buffer_pos * HEIGHT * WIDTH + index] = pos; - //pos.w = 1.0f; - - //float4 final_pos = mul(pose, pos); - - //// output - //output_ar[index] = final_pos.xyz; - - // optical - float2 optical; - optical.x = optical_ar[index]; - optical.y = optical_ar[index + WIDTH * HEIGHT]; - optical_buffer[buffer_pos * HEIGHT * WIDTH + index] = optical; - - // pose - pose_buffer[buffer_pos] = pose; - - output_ar[index] = pos; - - } -} - - -// EdgeDetection -[numthreads(16, 8, 1)] -void EdgeDetection(uint3 groupId : SV_GroupID, uint3 threadId : SV_GroupThreadID) -{ - int xIndex = groupId.x * 16 + threadId.x; - int yIndex = groupId.y * 8 + threadId.y; - - int index = xIndex + yIndex * WIDTH; - - if (xIndex > 1 && xIndex < WIDTH - 1 - 1 && yIndex > 1 && yIndex < HEIGHT - 1 - 1) - { - //float3 P = output_ar[index]; - //float3 Px_prev = output_ar[(xIndex - 1) + yIndex * WIDTH]; - //float3 Px_next = output_ar[(xIndex + 1) + yIndex * WIDTH]; - //float3 Py_prev = output_ar[xIndex + (yIndex - 1) * WIDTH]; - //float3 Py_next = output_ar[xIndex + (yIndex + 1) * WIDTH]; - - //// Compute gradients in each direction - //float3 gradX = length(Px_next - Px_prev); - //float3 gradY = length(Py_next - Py_prev); - - //// Calculate the gradient magnitude in 3D - //float gradientMagnitude = gradX * gradX + gradY * gradY; - - //// Apply threshold to detect edges - //if (sqrt(gradientMagnitude) > edgethreshold) - //{ - // output_ar[index] = float3(0.0f, 0.0f, 0.0f); - //} - } -} - -int calc_pre_buffer_pos(int pos) -{ - if (pos == 0) - { - return num_frames - 1; - } - else - { - return pos - 1; - } -} - -// EdgeDetection -[numthreads(16, 8, 1)] -void CVD(uint3 groupId : SV_GroupID, uint3 threadId : SV_GroupThreadID) -{ - int aimxIndex = groupId.x * 16 + threadId.x; - int aimyIndex = groupId.y * 8 + threadId.y; - - int aimindex = aimxIndex + aimyIndex * WIDTH; - - if (aimxIndex > 0 && aimxIndex < WIDTH - 1 && aimyIndex > 0 && aimyIndex < HEIGHT - 1 && depth_buffer[aimindex].z > 0.01f) - { - float3 aimPoint = depth_buffer[buffer_pos * HEIGHT * WIDTH + aimindex]; - float z = aimPoint.z; - float sumz = z; - float3 sumpoint = aimPoint; - float total_weight = 1.0f; - - - - [unroll] - for (int dx = -1; dx < 2; ++dx) - { - [unroll] - for (int dy = -1; dy < 2; ++dy) - { - - float3 prev_point = depth_buffer[buffer_pos * WIDTH * HEIGHT + aimxIndex + dx + (aimyIndex + dy) * WIDTH]; - - if (prev_point.z >= 0.01f) - { - float weight = exp(-cvd_weight * (max(aimPoint.z, prev_point.z) / min(aimPoint.z, prev_point.z) - 1.0f)); - total_weight = total_weight + weight; - sumpoint += prev_point * weight; - } - - } - } - - [unroll] - for (int dx1 = -1; dx1 < 2; ++dx1) - { - [unroll] - for (int dy1 = -1; dy1 < 2; ++dy1) - { - int xIndex = aimxIndex + dx1; - int yIndex = aimyIndex + dy1; - - int index = xIndex + yIndex * WIDTH; - int current_pos = buffer_pos; - - for (int i = 0; i < num_frames - 1; ++i) - { - int pre_pos = calc_pre_buffer_pos(current_pos); - float2 optical_flow = optical_buffer[current_pos * WIDTH * HEIGHT + xIndex + yIndex * WIDTH]; - - int prev_x = xIndex - int(optical_flow.x); - int prev_y = yIndex - int(optical_flow.y); - - if (prev_x > 0 && prev_x < WIDTH - 1 && prev_y > 0 && prev_y < HEIGHT - 1) - { - int prev_index = prev_x + prev_y * WIDTH; - - float3 prev_point = depth_buffer[pre_pos * WIDTH * HEIGHT + prev_index]; - float4x4 transform = mul(inverse_pose, pose_buffer[pre_pos]); - float3 mapped_point = mul(transform, float4(prev_point, 1.0f)); - - if (mapped_point.z >= 0.01f) - { - //float weight = 0.3f; - float ratio = max(aimPoint.z, mapped_point.z) / min(aimPoint.z, mapped_point.z) - 1.0f; - float weight = exp(-cvd_weight * ratio); - total_weight = total_weight + weight; - sumpoint += mapped_point * weight; - } - } - - xIndex = prev_x; - yIndex = prev_y; - current_pos = pre_pos; - } - - } - } - - float3 final_point = sumpoint / total_weight; - - if (cvd_weight < 100) - { - aimPoint = final_point; - } - - output_ar[aimindex] = aimPoint; - depth_buffer[buffer_pos * HEIGHT * WIDTH + aimindex] = aimPoint; - - } -} - -//// EdgeDetection -//[numthreads(16, 16, 1)] -//void CVD(uint3 groupId : SV_GroupID, uint3 threadId : SV_GroupThreadID) -//{ -// int xIndex = groupId.x * 16 + threadId.x; -// int yIndex = groupId.y * 16 + threadId.y; - -// int index = xIndex + yIndex * WIDTH; - -// if (xIndex > 45 && xIndex < WIDTH - 1 - 45 && yIndex > 45 && yIndex < HEIGHT - 1 - 45) -// { -// float3 currentPoint = output_ar[index]; - -// float3 sumPoints = currentPoint; -// float total_weight = 1.0f; - -// for (int i = 0; i < num_frames; i++) -// { -// int frameIndex = (buffer_pos - i - 1 + num_frames) % num_frames; - -// // extrinsic -// float4x4 historicalPose = pose_buffer[frameIndex]; -// float4x4 transform = mul(inverse_pose, historicalPose); -// float4 currentPoint4 = float4(currentPoint, 1.0); -// float4 transformedPoint4 = mul(transform, currentPoint4); - -// // intrinsic -// float fx = intrinsics.x; -// float fy = intrinsics.y; -// float cx = intrinsics.z; -// float cy = intrinsics.w; -// float x = transformedPoint4.x / transformedPoint4.z * fx + cx; -// float y = transformedPoint4.y / transformedPoint4.z * fy + cy;s -// int xPrev = (int) (x + 0.5); -// int yPrev = (int) (y + 0.5); - -// if (xPrev > 45 && xPrev < WIDTH - 1 - 45 && yPrev > 45 && yPrev < HEIGHT - 1 - 45) -// { -// int prevIndex = xPrev + yPrev * WIDTH + frameIndex * WIDTH * HEIGHT; - -// float3 historicalPoint = depth_buffer[prevIndex]; - -// if (historicalPoint.z > 0) -// { -// sumPoints += historicalPoint * 1.0f; -// total_weight = total_weight + 1.0f; -// } - -// } -// } - -// float3 final_point = sumPoints / total_weight; -// depth_buffer[buffer_pos * WIDTH * HEIGHT + index] = final_point; -// pose_buffer[buffer_pos] = pose; - -// output_ar[index] = mul(inverse_pose, float4(final_point, 1.0f)); - -// } -//} - - - - - - diff --git a/Assets/PointClouds/2poseCVDShaderRight.compute.meta b/Assets/PointClouds/2poseCVDShaderRight.compute.meta deleted file mode 100644 index c805aee..0000000 --- a/Assets/PointClouds/2poseCVDShaderRight.compute.meta +++ /dev/null @@ -1,7 +0,0 @@ -fileFormatVersion: 2 -guid: 2d18f6e47c4837f4e9e6a64c20f7ea6d -ComputeShaderImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Assets/PointClouds/InstancedIndirectColor.shader b/Assets/PointClouds/InstancedIndirectColor.shader index 513a44b..2274025 100644 --- a/Assets/PointClouds/InstancedIndirectColor.shader +++ b/Assets/PointClouds/InstancedIndirectColor.shader @@ -3,6 +3,7 @@ Shader "Custom/InstancedIndirectColor" { Tags { "RenderType" = "Opaque" } Pass { + Cull Off CGPROGRAM #pragma vertex vert #pragma fragment frag @@ -90,7 +91,7 @@ Shader "Custom/InstancedIndirectColor" { //float4 coor = {floor(instanceID * width), floor(instanceID * width), 0.0, 0.0}; float2 uv = TRANSFORM_TEX(coor.xy, _colorMap); - coor.x = uv.x; coor.y = uv.y; + coor.x = uv.x; coor.y = 1.0-uv.y; //float4 coor = {0.2,0.5,0.0,0.0}; o.color = tex2Dlod(_colorMap, coor); diff --git a/Assets/PointClouds/2poseCVDShaderLeft.compute b/Assets/PointClouds/poseCVDShader.compute similarity index 62% rename from Assets/PointClouds/2poseCVDShaderLeft.compute rename to Assets/PointClouds/poseCVDShader.compute index e13e717..48a38b6 100644 --- a/Assets/PointClouds/2poseCVDShaderLeft.compute +++ b/Assets/PointClouds/poseCVDShader.compute @@ -73,39 +73,6 @@ void Transformation(uint3 groupId : SV_GroupID, uint3 threadId : SV_GroupThreadI } } - -// EdgeDetection -[numthreads(16, 8, 1)] -void EdgeDetection(uint3 groupId : SV_GroupID, uint3 threadId : SV_GroupThreadID) -{ - int xIndex = groupId.x * 16 + threadId.x; - int yIndex = groupId.y * 8 + threadId.y; - - int index = xIndex + yIndex * WIDTH; - - if (xIndex > 1 && xIndex < WIDTH - 1 - 1 && yIndex > 1 && yIndex < HEIGHT - 1 - 1) - { - //float3 P = output_ar[index]; - //float3 Px_prev = output_ar[(xIndex - 1) + yIndex * WIDTH]; - //float3 Px_next = output_ar[(xIndex + 1) + yIndex * WIDTH]; - //float3 Py_prev = output_ar[xIndex + (yIndex - 1) * WIDTH]; - //float3 Py_next = output_ar[xIndex + (yIndex + 1) * WIDTH]; - - //// Compute gradients in each direction - //float3 gradX = length(Px_next - Px_prev); - //float3 gradY = length(Py_next - Py_prev); - - //// Calculate the gradient magnitude in 3D - //float gradientMagnitude = gradX * gradX + gradY * gradY; - - //// Apply threshold to detect edges - //if (sqrt(gradientMagnitude) > edgethreshold) - //{ - // output_ar[index] = float3(0.0f, 0.0f, 0.0f); - //} - } -} - int calc_pre_buffer_pos(int pos) { if (pos == 0) @@ -214,69 +181,3 @@ void CVD(uint3 groupId : SV_GroupID, uint3 threadId : SV_GroupThreadID) } } - -//// EdgeDetection -//[numthreads(16, 16, 1)] -//void CVD(uint3 groupId : SV_GroupID, uint3 threadId : SV_GroupThreadID) -//{ -// int xIndex = groupId.x * 16 + threadId.x; -// int yIndex = groupId.y * 16 + threadId.y; - -// int index = xIndex + yIndex * WIDTH; - -// if (xIndex > 45 && xIndex < WIDTH - 1 - 45 && yIndex > 45 && yIndex < HEIGHT - 1 - 45) -// { -// float3 currentPoint = output_ar[index]; - -// float3 sumPoints = currentPoint; -// float total_weight = 1.0f; - -// for (int i = 0; i < num_frames; i++) -// { -// int frameIndex = (buffer_pos - i - 1 + num_frames) % num_frames; - -// // extrinsic -// float4x4 historicalPose = pose_buffer[frameIndex]; -// float4x4 transform = mul(inverse_pose, historicalPose); -// float4 currentPoint4 = float4(currentPoint, 1.0); -// float4 transformedPoint4 = mul(transform, currentPoint4); - -// // intrinsic -// float fx = intrinsics.x; -// float fy = intrinsics.y; -// float cx = intrinsics.z; -// float cy = intrinsics.w; -// float x = transformedPoint4.x / transformedPoint4.z * fx + cx; -// float y = transformedPoint4.y / transformedPoint4.z * fy + cy;s -// int xPrev = (int) (x + 0.5); -// int yPrev = (int) (y + 0.5); - -// if (xPrev > 45 && xPrev < WIDTH - 1 - 45 && yPrev > 45 && yPrev < HEIGHT - 1 - 45) -// { -// int prevIndex = xPrev + yPrev * WIDTH + frameIndex * WIDTH * HEIGHT; - -// float3 historicalPoint = depth_buffer[prevIndex]; - -// if (historicalPoint.z > 0) -// { -// sumPoints += historicalPoint * 1.0f; -// total_weight = total_weight + 1.0f; -// } - -// } -// } - -// float3 final_point = sumPoints / total_weight; -// depth_buffer[buffer_pos * WIDTH * HEIGHT + index] = final_point; -// pose_buffer[buffer_pos] = pose; - -// output_ar[index] = mul(inverse_pose, float4(final_point, 1.0f)); - -// } -//} - - - - - - diff --git a/Assets/PointClouds/poseCVDShaderLeft.compute.meta b/Assets/PointClouds/poseCVDShader.compute.meta similarity index 100% rename from Assets/PointClouds/poseCVDShaderLeft.compute.meta rename to Assets/PointClouds/poseCVDShader.compute.meta diff --git a/Assets/PointClouds/poseCVDShaderLeft.compute b/Assets/PointClouds/poseCVDShaderLeft.compute deleted file mode 100644 index e13e717..0000000 --- a/Assets/PointClouds/poseCVDShaderLeft.compute +++ /dev/null @@ -1,282 +0,0 @@ -// Each #kernel tells which function to compile; you can have many kernels -#pragma kernel Transformation -#pragma kernel EdgeDetection -#pragma kernel CVD - -#define WIDTH 640 -#define HEIGHT 480 - -// Number of frames -int num_frames; - -// Buffers -RWStructuredBuffer depth_ar; -RWStructuredBuffer optical_ar; -RWStructuredBuffer output_ar; -RWStructuredBuffer depth_buffer; -RWStructuredBuffer optical_buffer; -RWStructuredBuffer pose_buffer; - -// pose mat -float4x4 pose; -float4x4 inverse_pose; - -// buffer pos -int buffer_pos; - -// intrinsics -float4 intrinsics; - -// edge detection -float edgethreshold; - -// weight -float cvd_weight; - -// EdgeDetection -[numthreads(16, 8, 1)] -void Transformation(uint3 groupId : SV_GroupID, uint3 threadId : SV_GroupThreadID) -{ - int xIndex = groupId.x * 16 + threadId.x; - int yIndex = groupId.y * 8 + threadId.y; - - int index = xIndex + yIndex * WIDTH; - - if (xIndex > 0 && xIndex < WIDTH - 1 && yIndex > 0 && yIndex < HEIGHT - 1) - { - uint depth_idx = (WIDTH * (HEIGHT - yIndex - 1)) + (WIDTH - xIndex - 1); - - float3 pos; - pos.z = depth_ar[index]; - pos.x = (xIndex - intrinsics.x) * pos.z / intrinsics.z; - pos.y = (yIndex - intrinsics.y) * pos.z / intrinsics.w; - - depth_buffer[buffer_pos * HEIGHT * WIDTH + index] = pos; - //pos.w = 1.0f; - - //float4 final_pos = mul(pose, pos); - - //// output - //output_ar[index] = final_pos.xyz; - - // optical - float2 optical; - optical.x = optical_ar[index]; - optical.y = optical_ar[index + WIDTH * HEIGHT]; - optical_buffer[buffer_pos * HEIGHT * WIDTH + index] = optical; - - // pose - pose_buffer[buffer_pos] = pose; - - output_ar[index] = pos; - - } -} - - -// EdgeDetection -[numthreads(16, 8, 1)] -void EdgeDetection(uint3 groupId : SV_GroupID, uint3 threadId : SV_GroupThreadID) -{ - int xIndex = groupId.x * 16 + threadId.x; - int yIndex = groupId.y * 8 + threadId.y; - - int index = xIndex + yIndex * WIDTH; - - if (xIndex > 1 && xIndex < WIDTH - 1 - 1 && yIndex > 1 && yIndex < HEIGHT - 1 - 1) - { - //float3 P = output_ar[index]; - //float3 Px_prev = output_ar[(xIndex - 1) + yIndex * WIDTH]; - //float3 Px_next = output_ar[(xIndex + 1) + yIndex * WIDTH]; - //float3 Py_prev = output_ar[xIndex + (yIndex - 1) * WIDTH]; - //float3 Py_next = output_ar[xIndex + (yIndex + 1) * WIDTH]; - - //// Compute gradients in each direction - //float3 gradX = length(Px_next - Px_prev); - //float3 gradY = length(Py_next - Py_prev); - - //// Calculate the gradient magnitude in 3D - //float gradientMagnitude = gradX * gradX + gradY * gradY; - - //// Apply threshold to detect edges - //if (sqrt(gradientMagnitude) > edgethreshold) - //{ - // output_ar[index] = float3(0.0f, 0.0f, 0.0f); - //} - } -} - -int calc_pre_buffer_pos(int pos) -{ - if (pos == 0) - { - return num_frames - 1; - } - else - { - return pos - 1; - } -} - -// EdgeDetection -[numthreads(16, 8, 1)] -void CVD(uint3 groupId : SV_GroupID, uint3 threadId : SV_GroupThreadID) -{ - int aimxIndex = groupId.x * 16 + threadId.x; - int aimyIndex = groupId.y * 8 + threadId.y; - - int aimindex = aimxIndex + aimyIndex * WIDTH; - - if (aimxIndex > 0 && aimxIndex < WIDTH - 1 && aimyIndex > 0 && aimyIndex < HEIGHT - 1 && depth_buffer[aimindex].z > 0.01f) - { - float3 aimPoint = depth_buffer[buffer_pos * HEIGHT * WIDTH + aimindex]; - float z = aimPoint.z; - float sumz = z; - float3 sumpoint = aimPoint; - float total_weight = 1.0f; - - - - [unroll] - for (int dx = -1; dx < 2; ++dx) - { - [unroll] - for (int dy = -1; dy < 2; ++dy) - { - - float3 prev_point = depth_buffer[buffer_pos * WIDTH * HEIGHT + aimxIndex + dx + (aimyIndex + dy) * WIDTH]; - - if (prev_point.z >= 0.01f) - { - float weight = exp(-cvd_weight * (max(aimPoint.z, prev_point.z) / min(aimPoint.z, prev_point.z) - 1.0f)); - total_weight = total_weight + weight; - sumpoint += prev_point * weight; - } - - } - } - - [unroll] - for (int dx1 = -1; dx1 < 2; ++dx1) - { - [unroll] - for (int dy1 = -1; dy1 < 2; ++dy1) - { - int xIndex = aimxIndex + dx1; - int yIndex = aimyIndex + dy1; - - int index = xIndex + yIndex * WIDTH; - int current_pos = buffer_pos; - - for (int i = 0; i < num_frames - 1; ++i) - { - int pre_pos = calc_pre_buffer_pos(current_pos); - float2 optical_flow = optical_buffer[current_pos * WIDTH * HEIGHT + xIndex + yIndex * WIDTH]; - - int prev_x = xIndex - int(optical_flow.x); - int prev_y = yIndex - int(optical_flow.y); - - if (prev_x > 0 && prev_x < WIDTH - 1 && prev_y > 0 && prev_y < HEIGHT - 1) - { - int prev_index = prev_x + prev_y * WIDTH; - - float3 prev_point = depth_buffer[pre_pos * WIDTH * HEIGHT + prev_index]; - float4x4 transform = mul(inverse_pose, pose_buffer[pre_pos]); - float3 mapped_point = mul(transform, float4(prev_point, 1.0f)); - - if (mapped_point.z >= 0.01f) - { - //float weight = 0.3f; - float ratio = max(aimPoint.z, mapped_point.z) / min(aimPoint.z, mapped_point.z) - 1.0f; - float weight = exp(-cvd_weight * ratio); - total_weight = total_weight + weight; - sumpoint += mapped_point * weight; - } - } - - xIndex = prev_x; - yIndex = prev_y; - current_pos = pre_pos; - } - - } - } - - float3 final_point = sumpoint / total_weight; - - if (cvd_weight < 100) - { - aimPoint = final_point; - } - - output_ar[aimindex] = aimPoint; - depth_buffer[buffer_pos * HEIGHT * WIDTH + aimindex] = aimPoint; - - } -} - -//// EdgeDetection -//[numthreads(16, 16, 1)] -//void CVD(uint3 groupId : SV_GroupID, uint3 threadId : SV_GroupThreadID) -//{ -// int xIndex = groupId.x * 16 + threadId.x; -// int yIndex = groupId.y * 16 + threadId.y; - -// int index = xIndex + yIndex * WIDTH; - -// if (xIndex > 45 && xIndex < WIDTH - 1 - 45 && yIndex > 45 && yIndex < HEIGHT - 1 - 45) -// { -// float3 currentPoint = output_ar[index]; - -// float3 sumPoints = currentPoint; -// float total_weight = 1.0f; - -// for (int i = 0; i < num_frames; i++) -// { -// int frameIndex = (buffer_pos - i - 1 + num_frames) % num_frames; - -// // extrinsic -// float4x4 historicalPose = pose_buffer[frameIndex]; -// float4x4 transform = mul(inverse_pose, historicalPose); -// float4 currentPoint4 = float4(currentPoint, 1.0); -// float4 transformedPoint4 = mul(transform, currentPoint4); - -// // intrinsic -// float fx = intrinsics.x; -// float fy = intrinsics.y; -// float cx = intrinsics.z; -// float cy = intrinsics.w; -// float x = transformedPoint4.x / transformedPoint4.z * fx + cx; -// float y = transformedPoint4.y / transformedPoint4.z * fy + cy;s -// int xPrev = (int) (x + 0.5); -// int yPrev = (int) (y + 0.5); - -// if (xPrev > 45 && xPrev < WIDTH - 1 - 45 && yPrev > 45 && yPrev < HEIGHT - 1 - 45) -// { -// int prevIndex = xPrev + yPrev * WIDTH + frameIndex * WIDTH * HEIGHT; - -// float3 historicalPoint = depth_buffer[prevIndex]; - -// if (historicalPoint.z > 0) -// { -// sumPoints += historicalPoint * 1.0f; -// total_weight = total_weight + 1.0f; -// } - -// } -// } - -// float3 final_point = sumPoints / total_weight; -// depth_buffer[buffer_pos * WIDTH * HEIGHT + index] = final_point; -// pose_buffer[buffer_pos] = pose; - -// output_ar[index] = mul(inverse_pose, float4(final_point, 1.0f)); - -// } -//} - - - - - - diff --git a/Assets/PointClouds/poseCVDShaderRight.compute b/Assets/PointClouds/poseCVDShaderRight.compute deleted file mode 100644 index e13e717..0000000 --- a/Assets/PointClouds/poseCVDShaderRight.compute +++ /dev/null @@ -1,282 +0,0 @@ -// Each #kernel tells which function to compile; you can have many kernels -#pragma kernel Transformation -#pragma kernel EdgeDetection -#pragma kernel CVD - -#define WIDTH 640 -#define HEIGHT 480 - -// Number of frames -int num_frames; - -// Buffers -RWStructuredBuffer depth_ar; -RWStructuredBuffer optical_ar; -RWStructuredBuffer output_ar; -RWStructuredBuffer depth_buffer; -RWStructuredBuffer optical_buffer; -RWStructuredBuffer pose_buffer; - -// pose mat -float4x4 pose; -float4x4 inverse_pose; - -// buffer pos -int buffer_pos; - -// intrinsics -float4 intrinsics; - -// edge detection -float edgethreshold; - -// weight -float cvd_weight; - -// EdgeDetection -[numthreads(16, 8, 1)] -void Transformation(uint3 groupId : SV_GroupID, uint3 threadId : SV_GroupThreadID) -{ - int xIndex = groupId.x * 16 + threadId.x; - int yIndex = groupId.y * 8 + threadId.y; - - int index = xIndex + yIndex * WIDTH; - - if (xIndex > 0 && xIndex < WIDTH - 1 && yIndex > 0 && yIndex < HEIGHT - 1) - { - uint depth_idx = (WIDTH * (HEIGHT - yIndex - 1)) + (WIDTH - xIndex - 1); - - float3 pos; - pos.z = depth_ar[index]; - pos.x = (xIndex - intrinsics.x) * pos.z / intrinsics.z; - pos.y = (yIndex - intrinsics.y) * pos.z / intrinsics.w; - - depth_buffer[buffer_pos * HEIGHT * WIDTH + index] = pos; - //pos.w = 1.0f; - - //float4 final_pos = mul(pose, pos); - - //// output - //output_ar[index] = final_pos.xyz; - - // optical - float2 optical; - optical.x = optical_ar[index]; - optical.y = optical_ar[index + WIDTH * HEIGHT]; - optical_buffer[buffer_pos * HEIGHT * WIDTH + index] = optical; - - // pose - pose_buffer[buffer_pos] = pose; - - output_ar[index] = pos; - - } -} - - -// EdgeDetection -[numthreads(16, 8, 1)] -void EdgeDetection(uint3 groupId : SV_GroupID, uint3 threadId : SV_GroupThreadID) -{ - int xIndex = groupId.x * 16 + threadId.x; - int yIndex = groupId.y * 8 + threadId.y; - - int index = xIndex + yIndex * WIDTH; - - if (xIndex > 1 && xIndex < WIDTH - 1 - 1 && yIndex > 1 && yIndex < HEIGHT - 1 - 1) - { - //float3 P = output_ar[index]; - //float3 Px_prev = output_ar[(xIndex - 1) + yIndex * WIDTH]; - //float3 Px_next = output_ar[(xIndex + 1) + yIndex * WIDTH]; - //float3 Py_prev = output_ar[xIndex + (yIndex - 1) * WIDTH]; - //float3 Py_next = output_ar[xIndex + (yIndex + 1) * WIDTH]; - - //// Compute gradients in each direction - //float3 gradX = length(Px_next - Px_prev); - //float3 gradY = length(Py_next - Py_prev); - - //// Calculate the gradient magnitude in 3D - //float gradientMagnitude = gradX * gradX + gradY * gradY; - - //// Apply threshold to detect edges - //if (sqrt(gradientMagnitude) > edgethreshold) - //{ - // output_ar[index] = float3(0.0f, 0.0f, 0.0f); - //} - } -} - -int calc_pre_buffer_pos(int pos) -{ - if (pos == 0) - { - return num_frames - 1; - } - else - { - return pos - 1; - } -} - -// EdgeDetection -[numthreads(16, 8, 1)] -void CVD(uint3 groupId : SV_GroupID, uint3 threadId : SV_GroupThreadID) -{ - int aimxIndex = groupId.x * 16 + threadId.x; - int aimyIndex = groupId.y * 8 + threadId.y; - - int aimindex = aimxIndex + aimyIndex * WIDTH; - - if (aimxIndex > 0 && aimxIndex < WIDTH - 1 && aimyIndex > 0 && aimyIndex < HEIGHT - 1 && depth_buffer[aimindex].z > 0.01f) - { - float3 aimPoint = depth_buffer[buffer_pos * HEIGHT * WIDTH + aimindex]; - float z = aimPoint.z; - float sumz = z; - float3 sumpoint = aimPoint; - float total_weight = 1.0f; - - - - [unroll] - for (int dx = -1; dx < 2; ++dx) - { - [unroll] - for (int dy = -1; dy < 2; ++dy) - { - - float3 prev_point = depth_buffer[buffer_pos * WIDTH * HEIGHT + aimxIndex + dx + (aimyIndex + dy) * WIDTH]; - - if (prev_point.z >= 0.01f) - { - float weight = exp(-cvd_weight * (max(aimPoint.z, prev_point.z) / min(aimPoint.z, prev_point.z) - 1.0f)); - total_weight = total_weight + weight; - sumpoint += prev_point * weight; - } - - } - } - - [unroll] - for (int dx1 = -1; dx1 < 2; ++dx1) - { - [unroll] - for (int dy1 = -1; dy1 < 2; ++dy1) - { - int xIndex = aimxIndex + dx1; - int yIndex = aimyIndex + dy1; - - int index = xIndex + yIndex * WIDTH; - int current_pos = buffer_pos; - - for (int i = 0; i < num_frames - 1; ++i) - { - int pre_pos = calc_pre_buffer_pos(current_pos); - float2 optical_flow = optical_buffer[current_pos * WIDTH * HEIGHT + xIndex + yIndex * WIDTH]; - - int prev_x = xIndex - int(optical_flow.x); - int prev_y = yIndex - int(optical_flow.y); - - if (prev_x > 0 && prev_x < WIDTH - 1 && prev_y > 0 && prev_y < HEIGHT - 1) - { - int prev_index = prev_x + prev_y * WIDTH; - - float3 prev_point = depth_buffer[pre_pos * WIDTH * HEIGHT + prev_index]; - float4x4 transform = mul(inverse_pose, pose_buffer[pre_pos]); - float3 mapped_point = mul(transform, float4(prev_point, 1.0f)); - - if (mapped_point.z >= 0.01f) - { - //float weight = 0.3f; - float ratio = max(aimPoint.z, mapped_point.z) / min(aimPoint.z, mapped_point.z) - 1.0f; - float weight = exp(-cvd_weight * ratio); - total_weight = total_weight + weight; - sumpoint += mapped_point * weight; - } - } - - xIndex = prev_x; - yIndex = prev_y; - current_pos = pre_pos; - } - - } - } - - float3 final_point = sumpoint / total_weight; - - if (cvd_weight < 100) - { - aimPoint = final_point; - } - - output_ar[aimindex] = aimPoint; - depth_buffer[buffer_pos * HEIGHT * WIDTH + aimindex] = aimPoint; - - } -} - -//// EdgeDetection -//[numthreads(16, 16, 1)] -//void CVD(uint3 groupId : SV_GroupID, uint3 threadId : SV_GroupThreadID) -//{ -// int xIndex = groupId.x * 16 + threadId.x; -// int yIndex = groupId.y * 16 + threadId.y; - -// int index = xIndex + yIndex * WIDTH; - -// if (xIndex > 45 && xIndex < WIDTH - 1 - 45 && yIndex > 45 && yIndex < HEIGHT - 1 - 45) -// { -// float3 currentPoint = output_ar[index]; - -// float3 sumPoints = currentPoint; -// float total_weight = 1.0f; - -// for (int i = 0; i < num_frames; i++) -// { -// int frameIndex = (buffer_pos - i - 1 + num_frames) % num_frames; - -// // extrinsic -// float4x4 historicalPose = pose_buffer[frameIndex]; -// float4x4 transform = mul(inverse_pose, historicalPose); -// float4 currentPoint4 = float4(currentPoint, 1.0); -// float4 transformedPoint4 = mul(transform, currentPoint4); - -// // intrinsic -// float fx = intrinsics.x; -// float fy = intrinsics.y; -// float cx = intrinsics.z; -// float cy = intrinsics.w; -// float x = transformedPoint4.x / transformedPoint4.z * fx + cx; -// float y = transformedPoint4.y / transformedPoint4.z * fy + cy;s -// int xPrev = (int) (x + 0.5); -// int yPrev = (int) (y + 0.5); - -// if (xPrev > 45 && xPrev < WIDTH - 1 - 45 && yPrev > 45 && yPrev < HEIGHT - 1 - 45) -// { -// int prevIndex = xPrev + yPrev * WIDTH + frameIndex * WIDTH * HEIGHT; - -// float3 historicalPoint = depth_buffer[prevIndex]; - -// if (historicalPoint.z > 0) -// { -// sumPoints += historicalPoint * 1.0f; -// total_weight = total_weight + 1.0f; -// } - -// } -// } - -// float3 final_point = sumPoints / total_weight; -// depth_buffer[buffer_pos * WIDTH * HEIGHT + index] = final_point; -// pose_buffer[buffer_pos] = pose; - -// output_ar[index] = mul(inverse_pose, float4(final_point, 1.0f)); - -// } -//} - - - - - - diff --git a/Assets/PointClouds/poseCVDShaderRight.compute.meta b/Assets/PointClouds/poseCVDShaderRight.compute.meta deleted file mode 100644 index b956d3c..0000000 --- a/Assets/PointClouds/poseCVDShaderRight.compute.meta +++ /dev/null @@ -1,7 +0,0 @@ -fileFormatVersion: 2 -guid: 51d1a422d7ec5fc4990391e880867eef -ComputeShaderImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Assets/PoseConsistentVideoDepth.cs b/Assets/PoseConsistentVideoDepth.cs index 4e15b9d..e4a761c 100644 --- a/Assets/PoseConsistentVideoDepth.cs +++ b/Assets/PoseConsistentVideoDepth.cs @@ -7,7 +7,6 @@ public class PoseConsistentVideoDepth : MonoBehaviour public ComputeShader pose_consistent_depth_shader; int transformation_kernel; - int edge_kernel; int cvd_kernel; int num_frames = 30; @@ -18,6 +17,8 @@ public class PoseConsistentVideoDepth : MonoBehaviour private ComputeBuffer depthReturnCompute; + private Vector4 intrinsics; + int buffer_pos = 0; int groupsX = (640 + 16 - 1) / 16; @@ -30,38 +31,22 @@ void Start() { // kernel transformation_kernel = pose_consistent_depth_shader.FindKernel("Transformation"); - edge_kernel = pose_consistent_depth_shader.FindKernel("EdgeDetection"); cvd_kernel = pose_consistent_depth_shader.FindKernel("CVD"); // Depth Buffer depthBufferCompute = new ComputeBuffer(480 * 640 * num_frames, sizeof(float) * 3); - pose_consistent_depth_shader.SetBuffer(transformation_kernel, "depth_buffer", depthBufferCompute); - //pose_consistent_depth_shader.SetBuffer(edge_kernel, "depth_buffer", depthBufferCompute); - pose_consistent_depth_shader.SetBuffer(cvd_kernel, "depth_buffer", depthBufferCompute); - // Pose Buffer poseBufferCompute = new ComputeBuffer(num_frames, sizeof(float) * 16); - //pose_consistent_depth_shader.SetBuffer(transformation_kernel, "pose_buffer", poseBufferCompute); - pose_consistent_depth_shader.SetBuffer(cvd_kernel, "pose_buffer", poseBufferCompute); - pose_consistent_depth_shader.SetBuffer(transformation_kernel, "pose_buffer", poseBufferCompute); // Optical Buffer opticalBufferCompute = new ComputeBuffer(480 * 640 * num_frames * 2, sizeof(float)); - pose_consistent_depth_shader.SetBuffer(cvd_kernel, "optical_buffer", opticalBufferCompute); - pose_consistent_depth_shader.SetBuffer(transformation_kernel, "optical_buffer", opticalBufferCompute); // Return Ar Buffer depthReturnCompute = new ComputeBuffer(480 * 640, sizeof(float) * 3); - pose_consistent_depth_shader.SetBuffer(transformation_kernel, "output_ar", depthReturnCompute); - pose_consistent_depth_shader.SetBuffer(edge_kernel, "output_ar", depthReturnCompute); - pose_consistent_depth_shader.SetBuffer(cvd_kernel, "output_ar", depthReturnCompute); - // others - pose_consistent_depth_shader.SetInt("num_frames", num_frames); - - Vector4 intr = new Vector4(CX, CY, FX, FY); - pose_consistent_depth_shader.SetVector("intrinsics", intr); + // Intrinsics + intrinsics = new Vector4(CX, CY, FX, FY); } // Update is called once per frame @@ -76,34 +61,43 @@ void OnDestroy() poseBufferCompute.Release(); } - public ComputeBuffer consistent_depth(ComputeBuffer depth_buffer, Matrix4x4 pose_mat, ComputeBuffer optical_buffer, bool activate_CVD, float edgethreshold, bool activate_edge_detection, bool activate_depth_completion, float cvd_weight) + public ComputeBuffer consistent_depth(ComputeBuffer depth_buffer, Matrix4x4 pose_mat, ComputeBuffer optical_buffer, bool activate_CVD, float edgethreshold, bool activate_depth_completion, float cvd_weight) { pose_consistent_depth_shader.SetInt("buffer_pos", buffer_pos); + pose_consistent_depth_shader.SetInt("num_frames", num_frames); + pose_consistent_depth_shader.SetVector("intrinsics", intrinsics); + //if (activate_CVD || activate_edge_detection || activate_depth_completion) if (true) { + pose_consistent_depth_shader.SetBuffer(transformation_kernel, "depth_ar", depth_buffer); pose_consistent_depth_shader.SetBuffer(transformation_kernel, "optical_ar", optical_buffer); + pose_consistent_depth_shader.SetBuffer(transformation_kernel, "output_ar", depthReturnCompute); + + pose_consistent_depth_shader.SetBuffer(transformation_kernel, "depth_buffer", depthBufferCompute); + pose_consistent_depth_shader.SetBuffer(transformation_kernel, "optical_buffer", opticalBufferCompute); + pose_consistent_depth_shader.SetBuffer(transformation_kernel, "pose_buffer", poseBufferCompute); + pose_consistent_depth_shader.SetMatrix("pose", pose_mat); - pose_consistent_depth_shader.SetBuffer(transformation_kernel, "depth_ar", depth_buffer); pose_consistent_depth_shader.Dispatch(transformation_kernel, groupsX, groupsY, 1); } - if (activate_edge_detection) - { - //pose_consistent_depth_shader.SetBuffer(edge_kernel, "depth_ar", depth_buffer); - pose_consistent_depth_shader.SetFloat("edgethreshold", edgethreshold); - pose_consistent_depth_shader.Dispatch(edge_kernel, groupsX, groupsY, 1); - } - pose_consistent_depth_shader.SetFloat("cvd_weight", cvd_weight); if (activate_CVD && cvd_weight < 99) { pose_consistent_depth_shader.SetBuffer(cvd_kernel, "depth_ar", depth_buffer); + pose_consistent_depth_shader.SetBuffer(cvd_kernel, "optical_ar", optical_buffer); + pose_consistent_depth_shader.SetBuffer(cvd_kernel, "output_ar", depthReturnCompute); + pose_consistent_depth_shader.SetBuffer(cvd_kernel, "depth_buffer", depthBufferCompute); + pose_consistent_depth_shader.SetBuffer(cvd_kernel, "optical_buffer", opticalBufferCompute); + pose_consistent_depth_shader.SetBuffer(cvd_kernel, "pose_buffer", poseBufferCompute); + + Matrix4x4 inverse_pose_mat = Matrix4x4.Inverse(pose_mat); pose_consistent_depth_shader.SetMatrix("inverse_pose", inverse_pose_mat); - pose_consistent_depth_shader.SetBuffer(cvd_kernel, "optical_ar", optical_buffer); pose_consistent_depth_shader.Dispatch(cvd_kernel, groupsX, groupsY, 1); + } diff --git a/Assets/Scenes/ghost.unity b/Assets/Scenes/ghost.unity index 6378977..3dc8a2b 100644 --- a/Assets/Scenes/ghost.unity +++ b/Assets/Scenes/ghost.unity @@ -2099,6 +2099,55 @@ Transform: - {fileID: 1241328620} m_Father: {fileID: 461408881} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!1 &259775552 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 259775553} + - component: {fileID: 259775554} + m_Layer: 0 + m_Name: CVD + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!4 &259775553 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 259775552} + serializedVersion: 2 + m_LocalRotation: {x: -0.8107805, y: -0.22419226, z: 0.13395992, w: 0.5238583} + m_LocalPosition: {x: -24.334625, y: 49.479366, z: -44.116913} + m_LocalScale: {x: 10, y: 10, z: 10} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 1527382482} + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!114 &259775554 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 259775552} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: d38c111f925931c44b6c2fe06434b60f, type: 3} + m_Name: + m_EditorClassIdentifier: + pose_consistent_depth_shader: {fileID: 7200000, guid: 4e3ab96e5c44e114987c2994dd32981a, type: 3} + CX: 318 + CY: 236 + FX: 329.64 + FY: 328.62 --- !u!1 &259812768 GameObject: m_ObjectHideFlags: 0 @@ -4196,6 +4245,10 @@ PrefabInstance: propertyPath: Mass value: 1.904699 objectReference: {fileID: 0} + - target: {fileID: 1137478775010377568, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: CVD + value: + objectReference: {fileID: 796725192} - target: {fileID: 1137478775010377568, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: width value: 640 @@ -4300,6 +4353,10 @@ PrefabInstance: propertyPath: m_IsActive value: 1 objectReference: {fileID: 0} + - target: {fileID: 1137478775515482002, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: CVD + value: + objectReference: {fileID: 1748619558} - target: {fileID: 1137478775515482002, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: y_min value: 0.5 @@ -4552,6 +4609,10 @@ PrefabInstance: propertyPath: t value: 1 objectReference: {fileID: 0} + - target: {fileID: 3310871052733856933, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: CVD + value: + objectReference: {fileID: 810747190} - target: {fileID: 3310871052733856933, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: y_min value: 0.5 @@ -4672,6 +4733,10 @@ PrefabInstance: propertyPath: t value: 1 objectReference: {fileID: 0} + - target: {fileID: 3310871053306987607, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + propertyPath: CVD + value: + objectReference: {fileID: 259775554} - target: {fileID: 3310871053306987607, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: y_min value: 0.5 @@ -5059,6 +5124,12 @@ PrefabInstance: - targetCorrespondingSourceObject: {fileID: 5050333314984075172, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} insertIndex: 0 addedObject: {fileID: 804843770} + - targetCorrespondingSourceObject: {fileID: 3699985937472072426, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + insertIndex: -1 + addedObject: {fileID: 796725191} + - targetCorrespondingSourceObject: {fileID: 2251966328704835437, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + insertIndex: -1 + addedObject: {fileID: 1748619557} - targetCorrespondingSourceObject: {fileID: 3310871052953474555, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} insertIndex: 0 addedObject: {fileID: 1834046043} @@ -5068,6 +5139,12 @@ PrefabInstance: - targetCorrespondingSourceObject: {fileID: 3310871052371914859, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} insertIndex: 0 addedObject: {fileID: 1444031539} + - targetCorrespondingSourceObject: {fileID: 1252929639408472541, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + insertIndex: -1 + addedObject: {fileID: 259775553} + - targetCorrespondingSourceObject: {fileID: 4430176667488560218, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + insertIndex: -1 + addedObject: {fileID: 810747191} - targetCorrespondingSourceObject: {fileID: 3310871052038647825, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} insertIndex: 5 addedObject: {fileID: 147311266} @@ -5186,7 +5263,7 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: UB_Model_file: C:/Users/brown/Documents/fmz/GHOST/Assets/onnx/promptda/prompt-depth-anything-vitl-rotated_optimized.onnx - UB_debug_dump: 1 + UB_debug_dump: 0 --- !u!4 &420483825 Transform: m_ObjectHideFlags: 0 @@ -7114,13 +7191,13 @@ MonoBehaviour: meanThreshold: 0.1 activate_CVD: 0 cvd_weight: 3 - show_spot: 1 + show_spot: 0 another_manager: {fileID: 1639569377} CVDLeft: {fileID: 305849529} CVDRight: {fileID: 1814602523} activate_depth_estimation: 1 activate_ICP: 0 - activate_edge_detection: 1 + activate_edge_detection: 0 edgeThreshold: 21 maxNeighbourNum: 10 AveragerLeft: {fileID: 1823001778} @@ -8795,6 +8872,55 @@ Transform: m_Children: [] m_Father: {fileID: 207396246} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!1 &796725190 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 796725191} + - component: {fileID: 796725192} + m_Layer: 0 + m_Name: CVD + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!4 &796725191 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 796725190} + serializedVersion: 2 + m_LocalRotation: {x: -0.8107805, y: -0.22419226, z: 0.13395992, w: 0.5238583} + m_LocalPosition: {x: -24.334625, y: 49.479366, z: -44.116913} + m_LocalScale: {x: 10, y: 10, z: 10} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 1965869693} + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!114 &796725192 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 796725190} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: d38c111f925931c44b6c2fe06434b60f, type: 3} + m_Name: + m_EditorClassIdentifier: + pose_consistent_depth_shader: {fileID: 7200000, guid: 4e3ab96e5c44e114987c2994dd32981a, type: 3} + CX: 318 + CY: 236 + FX: 329.64 + FY: 328.62 --- !u!1 &796748787 GameObject: m_ObjectHideFlags: 0 @@ -9001,6 +9127,55 @@ MonoBehaviour: Serializer: 0 protocol: 0 RosBridgeServerUrl: +--- !u!1 &810747189 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 810747191} + - component: {fileID: 810747190} + m_Layer: 0 + m_Name: CVD + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!114 &810747190 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 810747189} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: d38c111f925931c44b6c2fe06434b60f, type: 3} + m_Name: + m_EditorClassIdentifier: + pose_consistent_depth_shader: {fileID: 7200000, guid: 4e3ab96e5c44e114987c2994dd32981a, type: 3} + CX: 309 + CY: 236 + FX: 330.699 + FY: 329.77 +--- !u!4 &810747191 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 810747189} + serializedVersion: 2 + m_LocalRotation: {x: -0.804354, y: 0.24625555, z: -0.16830005, w: 0.51385605} + m_LocalPosition: {x: 33.941986, y: 43.365974, z: -44.120007} + m_LocalScale: {x: 10, y: 10, z: 10} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 1003514617} + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} --- !u!1 &819668449 GameObject: m_ObjectHideFlags: 0 @@ -10027,11 +10202,12 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: cdc798fc4b250c04b91b6a0767d25d31, type: 3} m_Name: m_EditorClassIdentifier: - RobotIP: 128.148.138.88 + RobotIP: 128.148.138.21 username: user password: bigbubbabigbubba useVisionPipeline: 1 - enableDebugDumps: 0 + depthCompletionModelFile: C:/Users/brown/Documents/fmz/GHOST/Assets/onnx/promptda/prompt-depth-anything-vitl-rotated_optimized.onnx + enableDebugDumps: 1 --- !u!1 &973382266 GameObject: m_ObjectHideFlags: 0 @@ -10230,6 +10406,11 @@ MonoBehaviour: FixInstruction: {fileID: 1060961219} spot1CurModeInstruct: {fileID: 1791865009} userCamera: {fileID: 771285273} +--- !u!4 &1003514617 stripped +Transform: + m_CorrespondingSourceObject: {fileID: 4430176667488560218, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + m_PrefabInstance: {fileID: 409472599} + m_PrefabAsset: {fileID: 0} --- !u!1 &1008755828 GameObject: m_ObjectHideFlags: 0 @@ -10383,10 +10564,11 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: cdc798fc4b250c04b91b6a0767d25d31, type: 3} m_Name: m_EditorClassIdentifier: - RobotIP: 128.148.138.87 + RobotIP: 128.148.138.22 username: user password: bigbubbabigbubba useVisionPipeline: 1 + depthCompletionModelFile: C:/Users/brown/Documents/fmz/GHOST/Assets/onnx/promptda/prompt-depth-anything-vitl-rotated_optimized.onnx enableDebugDumps: 0 --- !u!4 &1028337601 Transform: @@ -10463,7 +10645,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: d38c111f925931c44b6c2fe06434b60f, type: 3} m_Name: m_EditorClassIdentifier: - pose_consistent_depth_shader: {fileID: 7200000, guid: 77636f91f1ca1a642968ac70d7055d9f, type: 3} + pose_consistent_depth_shader: {fileID: 7200000, guid: 4e3ab96e5c44e114987c2994dd32981a, type: 3} CX: 318 CY: 236 FX: 329.64 @@ -11924,7 +12106,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: d38c111f925931c44b6c2fe06434b60f, type: 3} m_Name: m_EditorClassIdentifier: - pose_consistent_depth_shader: {fileID: 7200000, guid: 2d18f6e47c4837f4e9e6a64c20f7ea6d, type: 3} + pose_consistent_depth_shader: {fileID: 7200000, guid: 4e3ab96e5c44e114987c2994dd32981a, type: 3} CX: 309 CY: 236 FX: 330.699 @@ -14234,6 +14416,11 @@ Transform: m_Children: [] m_Father: {fileID: 0} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!4 &1527382482 stripped +Transform: + m_CorrespondingSourceObject: {fileID: 1252929639408472541, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + m_PrefabInstance: {fileID: 409472599} + m_PrefabAsset: {fileID: 0} --- !u!1 &1527530486 GameObject: m_ObjectHideFlags: 0 @@ -14341,6 +14528,11 @@ Transform: - {fileID: 1732706234} m_Father: {fileID: 57991046} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!4 &1544567470 stripped +Transform: + m_CorrespondingSourceObject: {fileID: 2251966328704835437, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + m_PrefabInstance: {fileID: 409472599} + m_PrefabAsset: {fileID: 0} --- !u!1 &1546403542 GameObject: m_ObjectHideFlags: 0 @@ -14927,7 +15119,7 @@ MonoBehaviour: meanThreshold: 0.1 activate_CVD: 0 cvd_weight: 3 - show_spot: 0 + show_spot: 1 another_manager: {fileID: 638278051} CVDLeft: {fileID: 1031760795} CVDRight: {fileID: 1274553787} @@ -15913,6 +16105,55 @@ AudioListener: m_PrefabAsset: {fileID: 0} m_GameObject: {fileID: 1747440746} m_Enabled: 1 +--- !u!1 &1748619556 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 1748619557} + - component: {fileID: 1748619558} + m_Layer: 0 + m_Name: CVD + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!4 &1748619557 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1748619556} + serializedVersion: 2 + m_LocalRotation: {x: -0.804354, y: 0.24625555, z: -0.16830005, w: 0.51385605} + m_LocalPosition: {x: 33.941986, y: 43.365974, z: -44.120007} + m_LocalScale: {x: 10, y: 10, z: 10} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 1544567470} + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!114 &1748619558 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1748619556} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: d38c111f925931c44b6c2fe06434b60f, type: 3} + m_Name: + m_EditorClassIdentifier: + pose_consistent_depth_shader: {fileID: 7200000, guid: 4e3ab96e5c44e114987c2994dd32981a, type: 3} + CX: 309 + CY: 236 + FX: 330.699 + FY: 329.77 --- !u!1 &1749052630 GameObject: m_ObjectHideFlags: 0 @@ -16673,7 +16914,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: d38c111f925931c44b6c2fe06434b60f, type: 3} m_Name: m_EditorClassIdentifier: - pose_consistent_depth_shader: {fileID: 7200000, guid: 51d1a422d7ec5fc4990391e880867eef, type: 3} + pose_consistent_depth_shader: {fileID: 7200000, guid: 4e3ab96e5c44e114987c2994dd32981a, type: 3} CX: 309 CY: 236 FX: 330.699 @@ -17629,6 +17870,11 @@ MonoBehaviour: scriptsOfModeToEnable: - {fileID: 1955072831} - {fileID: 1955072829} +--- !u!4 &1965869693 stripped +Transform: + m_CorrespondingSourceObject: {fileID: 3699985937472072426, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} + m_PrefabInstance: {fileID: 409472599} + m_PrefabAsset: {fileID: 0} --- !u!1 &1967322570 GameObject: m_ObjectHideFlags: 0 diff --git a/Assets/Scripts/DrawMeshInstanced.cs b/Assets/Scripts/DrawMeshInstanced.cs index a736d2d..d9612e2 100644 --- a/Assets/Scripts/DrawMeshInstanced.cs +++ b/Assets/Scripts/DrawMeshInstanced.cs @@ -46,6 +46,7 @@ public class DrawMeshInstanced : MonoBehaviour public RawImageSubscriber depthSubscriber; // ROS subscriber that holds the depth array public JPEGImageSubscriber colorSubscriber; // ROS subscriber holding the color image public SpotObserverClient spotObserverClient; + public PoseConsistentVideoDepth CVD; public bool savePointCloud; // allow user to save point cloud @@ -57,6 +58,8 @@ public class DrawMeshInstanced : MonoBehaviour private ComputeBuffer edge_buffer; private ComputeBuffer icp_res_buffer; + private ComputeBuffer optical_flow_buffer; + public Transform target; public Transform auxTarget; // In case someone changes the offset rotation @@ -209,7 +212,7 @@ private void Setup() globalProps = GetProperties(); - + icp_trans = Matrix4x4.identity; //inp_stm.Close(); @@ -243,6 +246,20 @@ private void Setup() meshPropertiesBuffer.SetData(globalProps); } + private ComputeBuffer process_depth(ComputeBuffer in_depth) + { + float edgethreshold = 0.0f; + + ComputeBuffer posed_depth; + + Matrix4x4 tform = get_current_pose(); + + // NOTE: Optical flow is not currently being used. + posed_depth = CVD.consistent_depth(in_depth, tform, optical_flow_buffer, false, edgethreshold, false, 0); + + return posed_depth; + } + private float get_target_rota() { //Debug.Log(convert_angle(target.eulerAngles.y).ToString() + " " + convert_angle(auxTarget.eulerAngles.y).ToString()); @@ -317,6 +334,8 @@ private void InitializeBuffers() edge_buffer = new ComputeBuffer((int)(480 * 640), sizeof(float)); depthBuffer.SetData(depth_ar); + optical_flow_buffer = new ComputeBuffer(480 * 640 * 2, sizeof(float)); + SetProperties(); SetGOPosition(); } @@ -406,8 +425,16 @@ private void UpdateTexture() for (int i = 0; i < 480 * 640; i++) { //depth_ar[i] = depth_ar_saved[i]; - depth_ar[i] = 2.0f; + depth_ar[i] = 0.0f; } + sparseBuffer.SetData(depth_ar); + for (int i = 0; i < 480 * 640; i++) + { + //depth_ar[i] = depth_ar_saved[i]; + depth_ar[i] = 1.0f; + } + depth_ar_buffer.SetData(depth_ar); + Debug.Log("SpotObserverCameraIndex: " + (int)SpotObserverCameraIndex); (color_image, depth_tensor) = spotObserverClient.GetCameraFeeds(SpotObserverCameraIndex); @@ -416,9 +443,14 @@ private void UpdateTexture() // HACK if (depth_tensor != null) { - ComputeTensorData.Pin(depth_tensor).buffer.GetData(depth_ar); + //ComputeTensorData.Pin(depth_tensor).buffer.GetData(depth_ar); + //depth_ar_buffer.SetData(depth_ar); + + depth_ar_buffer = ComputeTensorData.Pin(depth_tensor).buffer; } + depth_ar_buffer = process_depth(depth_ar_buffer); + new_depth_to_render = true; } else @@ -429,7 +461,8 @@ private void UpdateTexture() if (depth_image != null) { - depth_ar = depth_image.GetPixelData(0).ToArray(); + depth_ar_buffer.SetData(depth_ar); + //depth_ar = depth_image.GetPixelData(0).ToArray(); } Debug.Log("Depth image: " + (depth_image == null ? "null" : depth_image.width + "x" + depth_image.height)); //depth_ar = depthSubscriber.getDepthArr(); @@ -445,7 +478,7 @@ private void UpdateTexture() { return; } - sparseBuffer.SetData(avged_sparse); + //sparseBuffer.SetData(avged_sparse); calculate_icp = true; if (imageScriptIndex > 1) { calculate_icp = false; } // depth manager 2 @@ -488,10 +521,8 @@ private void UpdateTexture() //} - //temp_output_left = Averager.averaging(temp_output_left, is_not_moving, mean_averaging, median_averaging, edge_detection, edge_threshold); - (depth_ar_buffer, icp_trans, avged_sparse) = depthManager.update_depth_from_renderer(color_image, depth_ar, camera_index, calculate_icp, new_depth_to_render, depthManager.avg_before_completion); - + //(depth_ar_buffer, icp_trans, avged_sparse) = depthManager.update_depth_from_renderer(color_image, depth_ar, camera_index, calculate_icp, new_depth_to_render, depthManager.avg_before_completion); //if (depthManager.avg_before_completion) //{ // (depth_ar_buffer, icp_trans) = depthManager.update_depth_from_renderer(color_image, res_depth, camera_index, calculate_icp, new_depth_to_render); diff --git a/Assets/Scripts/SpotObserverClient.cs b/Assets/Scripts/SpotObserverClient.cs index 456a450..0e14715 100644 --- a/Assets/Scripts/SpotObserverClient.cs +++ b/Assets/Scripts/SpotObserverClient.cs @@ -93,14 +93,17 @@ private static void PluginLogCallback(string message) public string password; public bool useVisionPipeline = true; + public string depthCompletionModelFile; + public bool enableDebugDumps = false; private int robot_id = -1; private bool isConnected = false; + private bool isStreaming = false; + private bool isVisionPipelineRunning = false; private IntPtr model; private Texture2D[] rgb_textures; - private Texture2D[] depth_textures; //private Tensor[] rgb_tensors; private Tensor[] depth_tensors; @@ -114,42 +117,54 @@ private static void PluginLogCallback(string message) (uint)SpotCamera.FRONTLEFT, (uint)SpotCamera.FRONTRIGHT }; - - void Start() + + + private void launch_vision_pipeline() { - SOb_SetUnityLogCallback(PluginLogCallback); - RobotIP = RobotIP.Trim(); - robot_id = SOb_ConnectToSpot(RobotIP, username, password); - if (robot_id < 0) + if (model == IntPtr.Zero) { - Debug.LogError("Failed to connect to Spot robot at " + RobotIP); - isConnected = false; + Debug.LogError("No model loaded for vision pipeline."); return; } - Debug.Log("Connected to Spot robot at " + RobotIP); - isConnected = true; - - model = SOb_LoadModel("vision_pipeline.onnx", "cuda"); - if (model == IntPtr.Zero) + bool ret = SOb_LaunchVisionPipeline(robot_id, model); + if (!ret) { - Debug.LogError("Failed to load vision pipeline model."); - isConnected = false; + Debug.LogError("Failed to launch vision pipeline on Spot robot."); return; } + isVisionPipelineRunning = true; + } - bool ret = SOb_LaunchVisionPipeline(robot_id, model); + private void stop_vision_pipeline() + { + bool ret = SOb_StopVisionPipeline(robot_id); if (!ret) { - Debug.LogError("Failed to launch vision pipeline on Spot robot."); - isConnected = false; + Debug.LogError("Failed to stop vision pipeline on Spot robot."); return; } + isVisionPipelineRunning = false; + } + void Start() + { + SOb_SetUnityLogCallback(PluginLogCallback); if (enableDebugDumps) { SOb_ToggleDebugDumps("SPOT_OBSERVER_DUMPS"); } + RobotIP = RobotIP.Trim(); + robot_id = SOb_ConnectToSpot(RobotIP, username, password); + if (robot_id < 0) + { + Debug.LogError("Failed to connect to Spot robot at " + RobotIP); + isConnected = false; + return; + } + Debug.Log("Connected to Spot robot at " + RobotIP); + isConnected = true; + // Initialize camera resources // TODO: Unhardcode shapes TensorShape rgb_shape = new TensorShape(1, 3, 480, 640); // RGB shape @@ -159,7 +174,6 @@ void Start() // Initialize textures and tensors for each camera rgb_textures = new Texture2D[num_cams]; - depth_textures = new Texture2D[num_cams]; rgb_resources = new IntPtr[num_cams]; depth_resources = new IntPtr[num_cams]; @@ -181,11 +195,8 @@ void Start() for (var i = 0; i < num_cams; i++) { rgb_textures[i] = new Texture2D(640, 480, TextureFormat.RGB24, false); - //depth_textures[i] = new Texture2D(640, 480, TextureFormat.RFloat, false); rgb_resources[i] = rgb_textures[i].GetNativeTexturePtr(); - //depth_resources[i] = depth_textures[i].GetNativeTexturePtr(); - //rgb_resources[i] = ComputeTensorData.Pin(rgb_tensors[i]).buffer.GetNativeBufferPtr(); depth_tensors[i] = new Tensor(depth_shape); depth_resources[i] = ComputeTensorData.Pin(depth_tensors[i]).buffer.GetNativeBufferPtr(); @@ -202,86 +213,107 @@ void Start() if (!SOb_ReadCameraFeeds(robot_id, all_cams)) { Debug.LogError("Failed to read camera feeds from Spot robot."); - isConnected = false; + return; + } + isStreaming = true; + Debug.Log("Successfully started streaming camera feeds from Spot robot."); + + model = SOb_LoadModel(depthCompletionModelFile, "cuda"); + if (model == IntPtr.Zero) + { + Debug.LogError("Failed to load vision pipeline model."); return; } - if (!SOb_PushNextImageSetToUnityBuffers(robot_id)) + if (!isConnected) { - //Debug.LogError("Failed to read back image set to Unity buffers."); + SOb_DisconnectFromSpot(robot_id); + robot_id = -1; return; } + + launch_vision_pipeline(); + + + //if (!SOb_PushNextImageSetToUnityBuffers(robot_id)) + //{ + // //Debug.LogError("Failed to read back image set to Unity buffers."); + // return; + //} Debug.Log("Successfully registered and read camera feeds from Spot robot."); } void OnDestroy() { + Debug.Log("Disconnecting from Spot robot " + robot_id + ". isConnected = " + isConnected); + if (isConnected) { SOb_DisconnectFromSpot(robot_id); Debug.Log("Disconnected from Spot robot."); } - //// Dispose tensors - //foreach (var tensor in rgb_tensors) - //{ - // tensor.Dispose(); - //} - foreach (var tensor in depth_tensors) - { - tensor.Dispose(); - } - foreach (var texture in rgb_textures) - { - if (texture != null) + + if (depth_tensors != null) { + foreach (var tensor in depth_tensors) { - Destroy(texture); + tensor.Dispose(); } } - foreach (var texture in depth_textures) - { - if (texture != null) + if (rgb_textures != null) { + foreach (var texture in rgb_textures) { - Destroy(texture); + if (texture != null) + { + Destroy(texture); + } } } + + // TODO: Destroy the model + SpotCamToIdx.Clear(); } void Update() { - if (robot_id < 0) + if (robot_id < 0 || !isStreaming) { - Debug.LogError("Not connected to Spot robot. Cannot update."); return; } - //if (!SOb_PushNextImageSetToUnityBuffers(robot_id)) - //{ - // return; - //} - - if (!SOb_PushNextVisionPipelineImageSetToUnityBuffers(robot_id)) + if (useVisionPipeline) { - return; + if (!isVisionPipelineRunning) + { + launch_vision_pipeline(); + } + SOb_PushNextVisionPipelineImageSetToUnityBuffers(robot_id); + } else { + if (isVisionPipelineRunning) + { + stop_vision_pipeline(); + } + SOb_PushNextImageSetToUnityBuffers(robot_id); } - - - Debug.LogWarning("SUCCESSFULLY READ CAMERA FEEDS"); } public (Texture2D, Tensor) GetCameraFeeds(SpotCamera id) { if (!isConnected) { - Debug.LogError("Not connected to Spot robot. Cannot get camera feeds."); + Debug.LogError("Not connected to Spot robot. Cannot get camera feeds. (Robot ID " + robot_id + ")"); return (null, null); } + if (!isStreaming) { + Debug.LogError("Not streaming camera feeds for robot " + robot_id); + return (null, null); + } + if (!SpotCamToIdx.TryGetValue((int)id, out int idx)) { Debug.LogError("Invalid camera ID: " + (int)id); return (null, null); } - Debug.Log("Returning RGB and Depth for " + id); return (rgb_textures[idx], depth_tensors[idx]); } From 86eb0abc60cd4e1fcec235d91feab5752d749782 Mon Sep 17 00:00:00 2001 From: Faisal Zaghloul Date: Wed, 27 Aug 2025 17:13:09 -0400 Subject: [PATCH 8/8] New pipeline works reasonably well --- Assets/Scenes/ghost.unity | 22 +++--- Assets/Scripts/DrawMeshInstanced.cs | 102 ++++------------------------ 2 files changed, 25 insertions(+), 99 deletions(-) diff --git a/Assets/Scenes/ghost.unity b/Assets/Scenes/ghost.unity index 3dc8a2b..ac8adb1 100644 --- a/Assets/Scenes/ghost.unity +++ b/Assets/Scenes/ghost.unity @@ -4307,7 +4307,7 @@ PrefabInstance: objectReference: {fileID: 0} - target: {fileID: 1137478775010377568, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: use_saved_meshes - value: 1 + value: 0 objectReference: {fileID: 0} - target: {fileID: 1137478775010377568, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: spotObserverClient @@ -4415,7 +4415,7 @@ PrefabInstance: objectReference: {fileID: 0} - target: {fileID: 1137478775515482002, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: use_saved_meshes - value: 1 + value: 0 objectReference: {fileID: 0} - target: {fileID: 1137478775515482002, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: spotObserverClient @@ -4667,7 +4667,7 @@ PrefabInstance: objectReference: {fileID: 0} - target: {fileID: 3310871052733856933, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: use_saved_meshes - value: 1 + value: 0 objectReference: {fileID: 0} - target: {fileID: 3310871052733856933, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: spotObserverClient @@ -4791,7 +4791,7 @@ PrefabInstance: objectReference: {fileID: 0} - target: {fileID: 3310871053306987607, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: use_saved_meshes - value: 1 + value: 0 objectReference: {fileID: 0} - target: {fileID: 3310871053306987607, guid: 56ddb6be10446584fbc75833cadcd54f, type: 3} propertyPath: spotObserverClient @@ -5263,7 +5263,7 @@ MonoBehaviour: m_Name: m_EditorClassIdentifier: UB_Model_file: C:/Users/brown/Documents/fmz/GHOST/Assets/onnx/promptda/prompt-depth-anything-vitl-rotated_optimized.onnx - UB_debug_dump: 0 + UB_debug_dump: 1 --- !u!4 &420483825 Transform: m_ObjectHideFlags: 0 @@ -7191,13 +7191,13 @@ MonoBehaviour: meanThreshold: 0.1 activate_CVD: 0 cvd_weight: 3 - show_spot: 0 + show_spot: 1 another_manager: {fileID: 1639569377} CVDLeft: {fileID: 305849529} CVDRight: {fileID: 1814602523} - activate_depth_estimation: 1 + activate_depth_estimation: 0 activate_ICP: 0 - activate_edge_detection: 0 + activate_edge_detection: 1 edgeThreshold: 21 maxNeighbourNum: 10 AveragerLeft: {fileID: 1823001778} @@ -10206,8 +10206,8 @@ MonoBehaviour: username: user password: bigbubbabigbubba useVisionPipeline: 1 - depthCompletionModelFile: C:/Users/brown/Documents/fmz/GHOST/Assets/onnx/promptda/prompt-depth-anything-vitl-rotated_optimized.onnx - enableDebugDumps: 1 + depthCompletionModelFile: C:/Users/brown/Documents/fmz/GHOST/Assets/onnx/promptda/promptda-vitl-rotated_optimized.onnx + enableDebugDumps: 0 --- !u!1 &973382266 GameObject: m_ObjectHideFlags: 0 @@ -10568,7 +10568,7 @@ MonoBehaviour: username: user password: bigbubbabigbubba useVisionPipeline: 1 - depthCompletionModelFile: C:/Users/brown/Documents/fmz/GHOST/Assets/onnx/promptda/prompt-depth-anything-vitl-rotated_optimized.onnx + depthCompletionModelFile: C:/Users/brown/Documents/fmz/GHOST/Assets/onnx/promptda/promptda-vitl-rotated_optimized.onnx enableDebugDumps: 0 --- !u!4 &1028337601 Transform: diff --git a/Assets/Scripts/DrawMeshInstanced.cs b/Assets/Scripts/DrawMeshInstanced.cs index d9612e2..d27da9a 100644 --- a/Assets/Scripts/DrawMeshInstanced.cs +++ b/Assets/Scripts/DrawMeshInstanced.cs @@ -244,6 +244,13 @@ private void Setup() InitializeBuffers(); meshPropertiesBuffer.SetData(globalProps); + + // Hack: Initialize sparsebuffer to 0 + for (int i = 0; i < 480 * 640; i++) + { + depth_ar[i] = 0.0f; + } + sparseBuffer.SetData(depth_ar); } private ComputeBuffer process_depth(ComputeBuffer in_depth) @@ -414,8 +421,7 @@ private void UpdateTexture() material.SetVector("intrinsics", intr); Vector4 screenData = new Vector4((float)width, (float)height, 1 / (float)width, FY); - Debug.Log("Screen Data: " + screenData.ToString()); - Debug.Log("screenData[2] = " + screenData[2].ToString()); + depth_ar_buffer = process_depth(depth_ar_buffer); compute.SetVector("screenData", screenData); material.SetVector("screenData", screenData); @@ -428,13 +434,7 @@ private void UpdateTexture() depth_ar[i] = 0.0f; } sparseBuffer.SetData(depth_ar); - for (int i = 0; i < 480 * 640; i++) - { - //depth_ar[i] = depth_ar_saved[i]; - depth_ar[i] = 1.0f; - } - depth_ar_buffer.SetData(depth_ar); - + Debug.Log("SpotObserverCameraIndex: " + (int)SpotObserverCameraIndex); (color_image, depth_tensor) = spotObserverClient.GetCameraFeeds(SpotObserverCameraIndex); @@ -444,98 +444,24 @@ private void UpdateTexture() if (depth_tensor != null) { //ComputeTensorData.Pin(depth_tensor).buffer.GetData(depth_ar); - //depth_ar_buffer.SetData(depth_ar); - + depth_ar_buffer.SetData(depth_ar); depth_ar_buffer = ComputeTensorData.Pin(depth_tensor).buffer; } depth_ar_buffer = process_depth(depth_ar_buffer); - - new_depth_to_render = true; } else { - Debug.Log("Getting camera feeds from Spot Observer Client"); - //(color_image, depth_tensor) = spotObserverClient.GetCameraFeeds(0); - Debug.Log("Color image: " + (color_image == null ? "null" : color_image.width + "x" + color_image.height)); - + (color_image, depth_tensor) = spotObserverClient.GetCameraFeeds(SpotObserverCameraIndex); + if (depth_image != null) { depth_ar_buffer.SetData(depth_ar); - //depth_ar = depth_image.GetPixelData(0).ToArray(); - } - Debug.Log("Depth image: " + (depth_image == null ? "null" : depth_image.width + "x" + depth_image.height)); - //depth_ar = depthSubscriber.getDepthArr(); - if (depthSubscriber.new_depth == true) - { - new_depth_to_render = true; + depth_ar_buffer = ComputeTensorData.Pin(depth_tensor).buffer; } - depthSubscriber.new_depth = false; - } - Debug.Log("Depth array length: " + depth_ar.Length); - if (depth_ar.Length < 640 * 480) - { - return; + depth_ar_buffer = process_depth(depth_ar_buffer); } - //sparseBuffer.SetData(avged_sparse); - - calculate_icp = true; - if (imageScriptIndex > 1) { calculate_icp = false; } // depth manager 2 - - //if (depthManager.avg_before_completion) - //{ - // for (int j = 0; j < depth_ar.Length; j++) - // { - // depth_avg_buffer[buffer_pos, j] = depth_ar[j]; - // res_depth[j] = 0.0f; - // } - - // int num_valid = 0; - // float total_sum = 0.0f; - // for (int j = 0; j < 480 * 640; j++) - // { - // num_valid = 0; - // total_sum = 0.0f; - // for (int i = 0; i < 30; i++) - // { - - // if (depth_avg_buffer[i, j] >= 1.0f) - // { - // num_valid += 1; - // total_sum += depth_avg_buffer[i, j]; - // } - // } - // if (num_valid != 0) - // { - // res_depth[j] = total_sum / num_valid; - // } - // } - - - // buffer_pos += 1; - // if (buffer_pos >= 30) - // { - // buffer_pos = 0; - // } - //} - - - //temp_output_left = Averager.averaging(temp_output_left, is_not_moving, mean_averaging, median_averaging, edge_detection, edge_threshold); - //(depth_ar_buffer, icp_trans, avged_sparse) = depthManager.update_depth_from_renderer(color_image, depth_ar, camera_index, calculate_icp, new_depth_to_render, depthManager.avg_before_completion); - //if (depthManager.avg_before_completion) - //{ - // (depth_ar_buffer, icp_trans) = depthManager.update_depth_from_renderer(color_image, res_depth, camera_index, calculate_icp, new_depth_to_render); - //} - //else - //{ - // (depth_ar_buffer, icp_trans) = depthManager.update_depth_from_renderer(color_image, depth_ar, camera_index, calculate_icp, new_depth_to_render); - //} - - new_depth_to_render = false; - if (imageScriptIndex > 1) { icp_trans = Matrix4x4.identity; } // depth manager 2 - - } private void Update()