Search Unity

  1. Unity support for visionOS is now available. Learn more in our blog post.
    Dismiss Notice

Question How to Place ROS Image Messages in Scene and View in HMD?

Discussion in 'Robotics' started by Sehrja, Jun 21, 2022.

  1. Sehrja

    Sehrja

    Joined:
    Jun 13, 2017
    Posts:
    1
    Hey everyone, I had a quick question regarding the rendering of camera image messages from ROS in Unity. As of right now, I can get the images to render in the Unity editor as a widget, and the camera stream is working fine. However, I am not sure how to view the images in a VR HMD (Oculus 2). Within the headset the Image renderings in the game view (see attached image) are not visible to the HMD.

    Additionally, how do I place these stereo image textures in the virtual environment (like a projected video on a wall within the game)?

    Any help would be appreciated, thank you!

    Unity Robotics - Image Msg for HMD.png
     
    syedjawadakhtar likes this.
  2. maciejw94

    maciejw94

    Joined:
    Feb 11, 2022
    Posts:
    6
    This worked for me :)
    Code (CSharp):
    1. using System.Collections;
    2. using System.Collections.Generic;
    3. using UnityEngine;
    4. using UnityEngine.UI;
    5. using Unity.Robotics.ROSTCPConnector;
    6. // using RosMessageTypes.UnityRoboticsDemo;
    7. using RosMessageTypes.Sensor;
    8. using RosMessageTypes.Std;
    9. using RosMessageTypes.BuiltinInterfaces;
    10. using System;
    11. public class CamScript : MonoBehaviour
    12. {
    13.  
    14.     int currentCamIdx = 0;
    15.     // WebCamTexture tex;
    16.     Texture2D texRos;
    17.     public RawImage display;
    18.     ROSConnection m_Ros;
    19.     CompressedImageMsg img_msg;
    20.     string webcamiagetopic = "/webcam/image_raw";
    21.  
    22.     void Start() {
    23.         // start the ROS connection
    24.         m_Ros = ROSConnection.GetOrCreateInstance();
    25.         // register topic name
    26.         // m_Ros.RegisterPublisher<RosMessageTypes.Sensor.ImageMsg>(topicName);
    27.         Debug.Log(m_Ros.RosIPAddress);
    28.         m_Ros.Subscribe<ImageMsg>(webcamiagetopic, StartStopCam_Clicked);
    29.    
    30.     }
    31.    
    32.     public void SwapCam_Clicked() {
    33.         if (WebCamTexture.devices.Length > 0) {
    34.             currentCamIdx += 1;
    35.             currentCamIdx %= WebCamTexture.devices.Length;
    36.         }
    37.     }
    38.    
    39.     public void StartStopCam_Clicked(ImageMsg img) {
    40.         // stopping the prev output and clearing the texture
    41.         // if (texRos != null) {
    42.         //     display.texture = null;
    43.         //     // texRos.Stop();
    44.         //     texRos = null;
    45.         // } else {
    46.         // RenderTexture rendtextRos = new RenderTexture(640, 480, 0, UnityEngine.Experimental.Rendering.GraphicsFormat.R8G8B8A8_UNorm);
    47.         // rendtextRos.Create();
    48.         // rendtextRos.
    49.         texRos = new Texture2D((int) img.width, (int) img.height, TextureFormat.RGB24, false); // , TextureFormat.RGB24
    50.         BgrToRgb(img.data);
    51.         texRos.LoadRawTextureData(img.data);
    52.  
    53.         texRos.Apply();
    54.         display.texture = texRos;      
    55.     }
    56.  
    57.     public void BgrToRgb(byte[] data) {
    58.         for (int i = 0; i < data.Length; i += 3)
    59.         {
    60.             byte dummy = data[i];
    61.             data[i] = data[i + 2];
    62.             data[i + 2] = dummy;
    63.         }
    64.     }
    65. }
    66.  
     
    Last edited: Oct 24, 2022