Spot observer

.gitattributes

@@ -0,0 +1 @@
+Assets/onnx/promptda/prompt-depth-anything-vitl-rotated_optimized.onnx filter=lfs diff=lfs merge=lfs -text

Assets/CVDDataGenerator.cs

@@ -1,15 +1,15 @@
 using RosSharp.RosBridgeClient;
 using System.Collections;
 using System.Collections.Generic;
-using Unity.Sentis;
+
 using UnityEngine;
 
 public class CVDDataGenerator : MonoBehaviour
 {
     private bool first_run = true;
 
-    Tensor<float> previous_rgbL;
-    Tensor<float> previous_rgbR;
+    Unity.InferenceEngine.Tensor<float> previous_rgbL;
+    Unity.InferenceEngine.Tensor<float> previous_rgbR;
 
     //public BodyPoseSubscriber pose_subscriber;
 
@@ -133,20 +133,20 @@ IEnumerator SetFirstRunFalseAfterDelay(float delay)
 
     //}
 
-    public (ComputeBuffer, ComputeBuffer, Matrix4x4, Matrix4x4, ComputeBuffer, ComputeBuffer) generatePoseData(Tensor<float> depthL, Tensor<float> rgbL, Tensor<float> depthR, Tensor<float> rgbR, bool activate_depth_completion, bool activate_CVD)
+    public (ComputeBuffer, ComputeBuffer, Matrix4x4, Matrix4x4, ComputeBuffer, ComputeBuffer) generatePoseData(Unity.InferenceEngine.Tensor<float> depthL, Unity.InferenceEngine.Tensor<float> rgbL, Unity.InferenceEngine.Tensor<float> depthR, Unity.InferenceEngine.Tensor<float> rgbR, bool activate_depth_completion, bool activate_CVD)
     {
         if (first_run)
         {
-            previous_rgbL = new Tensor<float>(rgbL.shape);
-            previous_rgbR = new Tensor<float>(rgbR.shape);
+            previous_rgbL = new Unity.InferenceEngine.Tensor<float>(rgbL.shape);
+            previous_rgbR = new Unity.InferenceEngine.Tensor<float>(rgbR.shape);
 
             //buffer_depthL?.Release();
             //buffer_depthR?.Release();
             //buffer_opticalL?.Release();
             //buffer_opticalR?.Release();
 
-            buffer_depthL = ComputeTensorData.Pin(depthL).buffer;
-            buffer_depthR = ComputeTensorData.Pin(depthR).buffer;
+            buffer_depthL = Unity.InferenceEngine.ComputeTensorData.Pin(depthL).buffer;
+            buffer_depthR = Unity.InferenceEngine.ComputeTensorData.Pin(depthR).buffer;
             buffer_opticalL = new ComputeBuffer(480 * 640 * 2, sizeof(float));
             buffer_opticalR = new ComputeBuffer(480 * 640 * 2, sizeof(float));
 
@@ -167,8 +167,8 @@ IEnumerator SetFirstRunFalseAfterDelay(float delay)
             {
                 //buffer_depthL?.Release();
                 //buffer_depthR?.Release();
-                buffer_depthL = ComputeTensorData.Pin(depthL).buffer;
-                buffer_depthR = ComputeTensorData.Pin(depthR).buffer;
+                buffer_depthL = Unity.InferenceEngine.ComputeTensorData.Pin(depthL).buffer;
+                buffer_depthR = Unity.InferenceEngine.ComputeTensorData.Pin(depthR).buffer;
             }
 
 

Assets/DepthCompletion.cs

@@ -2,24 +2,24 @@
 using System.Collections;
 using System.Collections.Generic;
 using System.Runtime.InteropServices;
-using Unity.Sentis;
+
 using UnityEditor;
 using UnityEngine;
 using UnityEngine.VFX;
-using static Unity.Sentis.Model;
+using static Unity.InferenceEngine.Model;
 
 public class DepthCompletion : MonoBehaviour
 {
-    public ModelAsset rgbdModelAsset;
-    Model runtimeModelRGBD;
-    Worker workerRGBD;
+    public Unity.InferenceEngine.ModelAsset rgbdModelAsset;
+    Unity.InferenceEngine.Model runtimeModelRGBD;
+    Unity.InferenceEngine.Worker workerRGBD;
 
     public bool Use_UB; // Use UnityBYOM for inference
     public UBModel UB_Model; // Reference to UBModel script
 
     private int run_nconv;
 
-    Tensor<float> depth_outputTensor_0, depth_outputTensor_1;
+    Unity.InferenceEngine.Tensor<float> depth_outputTensor_0, depth_outputTensor_1;
     ComputeBuffer computeTensorData0, computeTensorData1;
 
 
@@ -29,8 +29,8 @@ public class DepthCompletion : MonoBehaviour
     void Start()
     {
         // Setup sentis model
-        runtimeModelRGBD = ModelLoader.Load(rgbdModelAsset);
-        workerRGBD = new Worker(runtimeModelRGBD, BackendType.GPUCompute);
+        runtimeModelRGBD = Unity.InferenceEngine.ModelLoader.Load(rgbdModelAsset);
+        workerRGBD = new Unity.InferenceEngine.Worker(runtimeModelRGBD, Unity.InferenceEngine.BackendType.GPUCompute);
 
         run_nconv = 2;
     }
@@ -47,7 +47,7 @@ void OnDestroy()
     // =============================================================================== //
     //                               Depth Completion                                  //
     // =============================================================================== //
-    public (ComputeBuffer, ComputeBuffer) complete(Tensor<float> depth_tensor_0, Tensor<float> color_tensor_0, Tensor<float> depth_tensor_1, Tensor<float> color_tensor_1)
+    public (ComputeBuffer, ComputeBuffer) complete(Unity.InferenceEngine.Tensor<float> depth_tensor_0, Unity.InferenceEngine.Tensor<float> color_tensor_0, Unity.InferenceEngine.Tensor<float> depth_tensor_1, Unity.InferenceEngine.Tensor<float> color_tensor_1)
     {
         if (Use_UB)
         {
@@ -88,11 +88,11 @@ void OnDestroy()
 
             workerRGBD.Schedule();
 
-            Tensor<float> depth_outputTensor_0 = workerRGBD.PeekOutput("output_depth_0") as Tensor<float>;
-            ComputeBuffer computeTensorData0 = ComputeTensorData.Pin(depth_outputTensor_0).buffer;
+            Unity.InferenceEngine.Tensor<float> depth_outputTensor_0 = workerRGBD.PeekOutput("output_depth_0") as Unity.InferenceEngine.Tensor<float>;
+            ComputeBuffer computeTensorData0 = Unity.InferenceEngine.ComputeTensorData.Pin(depth_outputTensor_0).buffer;
 
-            Tensor<float> depth_outputTensor_1 = workerRGBD.PeekOutput("output_depth_1") as Tensor<float>;
-            ComputeBuffer computeTensorData1 = ComputeTensorData.Pin(depth_outputTensor_1).buffer;
+            Unity.InferenceEngine.Tensor<float> depth_outputTensor_1 = workerRGBD.PeekOutput("output_depth_1") as Unity.InferenceEngine.Tensor<float>;
+            ComputeBuffer computeTensorData1 = Unity.InferenceEngine.ComputeTensorData.Pin(depth_outputTensor_1).buffer;
 
             return (computeTensorData0, computeTensorData1);
         }

Assets/DepthManager.cs

@@ -2,7 +2,8 @@
 using System.Collections.Generic;
 using UnityEngine;
 using UnityEngine.UI;
-using Unity.Sentis;
+using Unity.InferenceEngine;
+
 using Unity.VisualScripting;
 
 public class DepthManager : MonoBehaviour
@@ -29,11 +30,11 @@ public class DepthManager : MonoBehaviour
 
     public DepthAveraging AveragerLeft, AveragerRight;
 
-    private Tensor<float> depth_left_t_1;
-    private Tensor<float> rgb_left_t_1;
+    private Unity.InferenceEngine.Tensor<float> depth_left_t_1;
+    private Unity.InferenceEngine.Tensor<float> rgb_left_t_1;
 
-    private Tensor<float> depth_right_t_1;
-    private Tensor<float> rgb_right_t_1;
+    private Unity.InferenceEngine.Tensor<float> depth_right_t_1;
+    private Unity.InferenceEngine.Tensor<float> rgb_right_t_1;
 
     private bool received_left_1 = false;
     private bool received_right_1 = false;
@@ -74,8 +75,8 @@ public class DepthManager : MonoBehaviour
 
     private ComputeBuffer final_out;
 
-    TensorShape depth_shape = new TensorShape(1, 1, 480, 640);
-    TensorShape color_shape = new TensorShape(1, 3, 480, 640);
+    Unity.InferenceEngine.TensorShape depth_shape = new Unity.InferenceEngine.TensorShape(1, 1, 480, 640);
+    Unity.InferenceEngine.TensorShape color_shape = new Unity.InferenceEngine.TensorShape(1, 3, 480, 640);
 
     Matrix4x4 ICP_trans = Matrix4x4.identity;
     Matrix4x4 icp_trans_temp = Matrix4x4.identity;
@@ -125,11 +126,11 @@ void Start()
         //left_eye_data_timer_id = fps_timer.registerTimer("Left eye data");
         //right_eye_data_timer_id = fps_timer.registerTimer("Right eye data");
 
-        depth_left_t_1 = new Tensor<float>(depth_shape);
-        depth_right_t_1 = new Tensor<float>(depth_shape);
+        depth_left_t_1 = new Unity.InferenceEngine.Tensor<float>(depth_shape);
+        depth_right_t_1 = new Unity.InferenceEngine.Tensor<float>(depth_shape);
 
-        rgb_left_t_1 = new Tensor<float>(color_shape, data: null);
-        rgb_right_t_1 = new Tensor<float>(color_shape, data: null);
+        rgb_left_t_1 = new Unity.InferenceEngine.Tensor<float>(color_shape, data: null);
+        rgb_right_t_1 = new Unity.InferenceEngine.Tensor<float>(color_shape, data: null);
 
         received_left_1 = false;
         received_right_1 = false;
@@ -237,10 +238,10 @@ private IEnumerator ResetActivateDepthEstimation()
             }
             else
             {
-                depth_left_t_1 = new Tensor<float>(depth_shape, left_depth_avg);
+                depth_left_t_1 = new Unity.InferenceEngine.Tensor<float>(depth_shape, left_depth_avg);
                 //depth_left_t_1 = new Tensor<float>(depth_shape, depth);
             }
-            TextureConverter.ToTensor(rgb, rgb_left_t_1, tform);
+            Unity.InferenceEngine.TextureConverter.ToTensor(rgb, rgb_left_t_1, tform);
             rgb_left_t_1.Reshape(color_shape);
 
             received_left_1 = true;
@@ -259,10 +260,10 @@ private IEnumerator ResetActivateDepthEstimation()
             }
             else
             {
-                depth_right_t_1 = new Tensor<float>(depth_shape, right_depth_avg);
+                depth_right_t_1 = new Unity.InferenceEngine.Tensor<float>(depth_shape, right_depth_avg);
                 //depth_right_t_1 = new Tensor<float>(depth_shape, depth);
             }
-            TextureConverter.ToTensor(rgb, rgb_right_t_1, tform);
+            Unity.InferenceEngine.TextureConverter.ToTensor(rgb, rgb_right_t_1, tform);
             rgb_right_t_1.Reshape(color_shape);
 
             received_right_1 = true;
@@ -282,7 +283,7 @@ private IEnumerator ResetActivateDepthEstimation()
         return (temp_output_left_1, icp_trans_temp, right_depth_avg);
     }
 
-    private (ComputeBuffer, ComputeBuffer, Matrix4x4) process_depth(Tensor<float> depthL_1, Tensor<float> rgbL_1, Tensor<float> depthR_1, Tensor<float> rgbR_1, bool is_not_moving, bool calculate_icp)
+    private (ComputeBuffer, ComputeBuffer, Matrix4x4) process_depth(Unity.InferenceEngine.Tensor<float> depthL_1, Unity.InferenceEngine.Tensor<float> rgbL_1, Unity.InferenceEngine.Tensor<float> depthR_1, Unity.InferenceEngine.Tensor<float> rgbR_1, bool is_not_moving, bool calculate_icp)
     {
         ICP_trans = Matrix4x4.identity;
         if (calculate_icp && activate_ICP) 
@@ -294,8 +295,8 @@ private IEnumerator ResetActivateDepthEstimation()
 
 
         // Get compute buffers for depth
-        ComputeBuffer depthL_1_buf = ComputeTensorData.Pin(depthL_1).buffer;
-        ComputeBuffer depthR_1_buf = ComputeTensorData.Pin(depthR_1).buffer;
+        ComputeBuffer depthL_1_buf = Unity.InferenceEngine.ComputeTensorData.Pin(depthL_1).buffer;
+        ComputeBuffer depthR_1_buf = Unity.InferenceEngine.ComputeTensorData.Pin(depthR_1).buffer;
 
         AveragerLeft.prev_filling(depthL_1_buf);
         AveragerRight.prev_filling(depthR_1_buf);
@@ -306,8 +307,8 @@ private IEnumerator ResetActivateDepthEstimation()
         AveragerRight.update_depth_buffer(temp_depth_right);
 
 
-        temp_depth_left_return = CVDLeft.consistent_depth(temp_depth_left, mat_l, temp_optical_left, activate_CVD && is_not_moving, edgethreshold, activate_edge_detection, activate_depth_estimation, cvd_weight);
-        temp_depth_right_return = CVDRight.consistent_depth(temp_depth_right, mat_r, temp_optical_right, activate_CVD && is_not_moving, edgethreshold, activate_edge_detection, activate_depth_estimation, cvd_weight);
+        temp_depth_left_return = CVDLeft.consistent_depth(temp_depth_left, mat_l, temp_optical_left, activate_CVD && is_not_moving, edgethreshold, activate_depth_estimation, cvd_weight);
+        temp_depth_right_return = CVDRight.consistent_depth(temp_depth_right, mat_r, temp_optical_right, activate_CVD && is_not_moving, edgethreshold, activate_depth_estimation, cvd_weight);
 
         return (temp_depth_left_return, temp_depth_right_return, ICP_trans);
     }

Assets/ICPLauncher.cs

@@ -7,7 +7,7 @@
 using MathNet.Numerics.LinearAlgebra;
 using MathNet.Numerics.LinearAlgebra.Single;
 using Meta.WitAi;
-using Unity.Sentis;
+
 using System.IO;
 using System;
 
@@ -56,7 +56,7 @@ public class ICPLauncher : MonoBehaviour
     float3x3 R_all;
     float3 t_all;
 
-    TensorShape depth_shape;
+    Unity.InferenceEngine.TensorShape depth_shape;
 
     float global_min_error = 10000000000.0f;
 
@@ -119,7 +119,7 @@ void Start()
     {
         depth3d_return = new float3[W * H];
 
-        depth_shape = new TensorShape(1, 1, 480, 640);
+        depth_shape = new Unity.InferenceEngine.TensorShape(1, 1, 480, 640);
         buffer0 = new ComputeBuffer(480 * 640, sizeof(float));
         buffer1 = new ComputeBuffer(480 * 640, sizeof(float));
         buffer2 = new ComputeBuffer(480 * 640, sizeof(float));

Assets/OpticalFlowEstimator.cs

@@ -1,25 +1,25 @@
 using System.Collections;
 using System.Collections.Generic;
-using Unity.Sentis;
+
 using UnityEngine;
 
 public class OpticalFlowEstimator : MonoBehaviour
 {
-    public ModelAsset NeuV2;
-    Model runtimeModelNeuV2;
-    Worker workerNeuV2R;
-    Worker workerNeuV2L;
+    public Unity.InferenceEngine.ModelAsset NeuV2;
+    Unity.InferenceEngine.Model runtimeModelNeuV2;
+    Unity.InferenceEngine.Worker workerNeuV2R;
+    Unity.InferenceEngine.Worker workerNeuV2L;
 
     //ComputeBuffer computeTensorData0;
     //ComputeBuffer computeTensorData1;
 
     // Start is called before the first frame update
     void Start()
     {
-        runtimeModelNeuV2 = ModelLoader.Load(NeuV2);
-        workerNeuV2R = new Worker(runtimeModelNeuV2, BackendType.GPUCompute);
+        runtimeModelNeuV2 = Unity.InferenceEngine.ModelLoader.Load(NeuV2);
+        workerNeuV2R = new Unity.InferenceEngine.Worker(runtimeModelNeuV2, Unity.InferenceEngine.BackendType.GPUCompute);
 
-        workerNeuV2L = new Worker(runtimeModelNeuV2, BackendType.GPUCompute);
+        workerNeuV2L = new Unity.InferenceEngine.Worker(runtimeModelNeuV2, Unity.InferenceEngine.BackendType.GPUCompute);
     }
 
     void OnDestroy()
@@ -38,27 +38,27 @@ void Update()
 
     }
 
-    public (ComputeBuffer, ComputeBuffer) estimate_all(Tensor<float> rgbL0, Tensor<float> rgbL1, Tensor<float> rgbR0, Tensor<float> rgbR1)
+    public (ComputeBuffer, ComputeBuffer) estimate_all(Unity.InferenceEngine.Tensor<float> rgbL0, Unity.InferenceEngine.Tensor<float> rgbL1, Unity.InferenceEngine.Tensor<float> rgbR0, Unity.InferenceEngine.Tensor<float> rgbR1)
     {
         //Debug.Log("estimate Flow");
         // left
         workerNeuV2L.SetInput("input1", rgbL0);
         workerNeuV2L.SetInput("input2", rgbL1);
         workerNeuV2L.Schedule();
 
-        Tensor<float> depth_outputTensor_0 = workerNeuV2L.PeekOutput() as Tensor<float>;
+        Unity.InferenceEngine.Tensor<float> depth_outputTensor_0 = workerNeuV2L.PeekOutput() as Unity.InferenceEngine.Tensor<float>;
         //depth_outputTensor_0.Reshape(new TensorShape(2, 480, 640));
-        ComputeBuffer computeData0 = ComputeTensorData.Pin(depth_outputTensor_0).buffer;
+        ComputeBuffer computeData0 = Unity.InferenceEngine.ComputeTensorData.Pin(depth_outputTensor_0).buffer;
         //Debug.Log(depth_outputTensor_0);
 
         // right
         workerNeuV2R.SetInput("input1", rgbR0);
         workerNeuV2R.SetInput("input2", rgbR1);
         workerNeuV2R.Schedule();
 
-        Tensor<float> depth_outputTensor_1 = workerNeuV2R.PeekOutput() as Tensor<float>;
+        Unity.InferenceEngine.Tensor<float> depth_outputTensor_1 = workerNeuV2R.PeekOutput() as Unity.InferenceEngine.Tensor<float>;
         //depth_outputTensor_1.Reshape(new TensorShape(2, 480, 640));
-        ComputeBuffer computeData1 = ComputeTensorData.Pin(depth_outputTensor_1).buffer;
+        ComputeBuffer computeData1 = Unity.InferenceEngine.ComputeTensorData.Pin(depth_outputTensor_1).buffer;
         //Debug.Log(depth_outputTensor_1);
 
         //depth_outputTensor_0.Dispose();