Question How to Place ROS Image Messages in Scene and View in HMD?

Hey everyone, I had a quick question regarding the rendering of camera image messages from ROS in Unity. As of right now, I can get the images to render in the Unity editor as a widget, and the camera stream is working fine. However, I am not sure how to view the images in a VR HMD (Oculus 2). Within the headset the Image renderings in the game view (see attached image) are not visible to the HMD.

Additionally, how do I place these stereo image textures in the virtual environment (like a projected video on a wall within the game)?

Any help would be appreciated, thank you!

 

This worked for me

Code (CSharp):

1. using System.Collections;
2. using System.Collections.Generic;
3. using UnityEngine;
4. using UnityEngine.UI;
5. using Unity.Robotics.ROSTCPConnector;
6. // using RosMessageTypes.UnityRoboticsDemo;
7. using RosMessageTypes.Sensor;
8. using RosMessageTypes.Std;
9. using RosMessageTypes.BuiltinInterfaces;
10. using System;
11. public class CamScript : MonoBehaviour
12. {
13.  
14.     int currentCamIdx = 0;
15.     // WebCamTexture tex;
16.     Texture2D texRos;
17.     public RawImage display;
18.     ROSConnection m_Ros;
19.     CompressedImageMsg img_msg;
20.     string webcamiagetopic = "/webcam/image_raw";
21.  
22.     void Start() {
23.         // start the ROS connection
24.         m_Ros = ROSConnection.GetOrCreateInstance();
25.         // register topic name
26.         // m_Ros.RegisterPublisher<RosMessageTypes.Sensor.ImageMsg>(topicName);
27.         Debug.Log(m_Ros.RosIPAddress);
28.         m_Ros.Subscribe<ImageMsg>(webcamiagetopic, StartStopCam_Clicked);
29.    
30.     }
31.    
32.     public void SwapCam_Clicked() {
33.         if (WebCamTexture.devices.Length > 0) {
34.             currentCamIdx += 1;
35.             currentCamIdx %= WebCamTexture.devices.Length;
36.         }
37.     }
38.    
39.     public void StartStopCam_Clicked(ImageMsg img) {
40.         // stopping the prev output and clearing the texture
41.         // if (texRos != null) {
42.         //     display.texture = null;
43.         //     // texRos.Stop();
44.         //     texRos = null;
45.         // } else {
46.         // RenderTexture rendtextRos = new RenderTexture(640, 480, 0, UnityEngine.Experimental.Rendering.GraphicsFormat.R8G8B8A8_UNorm);
47.         // rendtextRos.Create();
48.         // rendtextRos.
49.         texRos = new Texture2D((int) img.width, (int) img.height, TextureFormat.RGB24, false); // , TextureFormat.RGB24
50.         BgrToRgb(img.data);
51.         texRos.LoadRawTextureData(img.data);
52.  
53.         texRos.Apply();
54.         display.texture = texRos;      
55.     }
56.  
57.     public void BgrToRgb(byte[] data) {
58.         for (int i = 0; i < data.Length; i += 3)
59.         {
60.             byte dummy = data[i];
61.             data[i] = data[i + 2];
62.             data[i + 2] = dummy;
63.         }
64.     }
65. }
66.