Search Unity

  1. Unity Asset Manager is now available in public beta. Try it out now and join the conversation here in the forums.
    Dismiss Notice
  2. Unity 6 Preview is now available. To find out what's new, have a look at our Unity 6 Preview blog post.
    Dismiss Notice
  3. Unity is excited to announce that we will be collaborating with TheXPlace for a summer game jam from June 13 - June 19. Learn more.
    Dismiss Notice

GetRawTextureData() Method resulting in flipped image in RViz

Discussion in 'Robotics' started by MrMagicMoz, Jul 29, 2021.

  1. MrMagicMoz

    MrMagicMoz

    Joined:
    Feb 7, 2021
    Posts:
    2
    I am currently publishing an image to ROS from my unity program, with the hope of tracking an AprilTag in the scene. However when viewing the image in RViz the image is flipped along its x axis and the AprilTag package cannot track the tag unless I flip the tag in the scene. What could be problem here ?

    Code (CSharp):
    1. using UnityEngine;
    2. using Unity.Robotics.ROSTCPConnector;
    3. using RosMessageTypes.UnityRoboticsDemo;
    4. using RosMessageTypes.Sensor;
    5. using RosMessageTypes.Std;
    6. using RosMessageTypes.BuiltinInterfaces;
    7. using System.Collections;
    8.  
    9. /// <summary>
    10. ///
    11. /// </summary>
    12.  
    13. [RequireComponent(typeof(ROSClockSubscriber))]
    14. public class ROSCameraDataPublisher : MonoBehaviour
    15. {
    16.     ROSConnection ros;
    17.     public string imageTopic = "/camera_rect/image_rect";
    18.     public string camInfoTopic = "/camera_rect/camera_info";
    19.  
    20.     public string CompressedImageTopic = "/camera_rect/image_rect_compressed";
    21.  
    22.     public Camera target_camera;
    23.  
    24.     public bool compressed = false;
    25.  
    26.     public float pubMsgFrequency = 30f;
    27.  
    28.     private float timeElapsed;
    29.     private RenderTexture renderTexture;
    30.     private RenderTexture lastTexture;
    31.  
    32.     private Texture2D mainCameraTexture;
    33.     private Rect frame;
    34.  
    35.  
    36.     private int frame_width;
    37.     private int frame_height;
    38.     private const int isBigEndian = 0;
    39.     private uint image_step = 4;
    40.     TimeMsg lastTime;
    41.  
    42.     private ROSClockSubscriber clock;
    43.  
    44.     private ImageMsg img_msg;
    45.     private CameraInfoMsg infoCamera;
    46.  
    47.     private HeaderMsg header;
    48.  
    49.     void Start()
    50.     {
    51.         // start the ROS connection
    52.         ros = ROSConnection.instance;
    53.  
    54.         if(ros)
    55.         {
    56.             ros.RegisterPublisher<ImageMsg>(imageTopic);
    57.             ros.RegisterPublisher<CompressedImageMsg>(CompressedImageTopic);
    58.  
    59.             ros.RegisterPublisher<CameraInfoMsg>(camInfoTopic);
    60.             clock = GetComponent<ROSClockSubscriber>();
    61.         }
    62.         else
    63.         {
    64.             Debug.Log("No ros connection found.");
    65.         }
    66.  
    67.  
    68.         if (!target_camera)
    69.         {
    70.             target_camera = Camera.main;
    71.         }
    72.  
    73.         if (Camera.main)
    74.         {
    75.             renderTexture = new RenderTexture(Camera.main.pixelWidth, Camera.main.pixelHeight, 0, UnityEngine.Experimental.Rendering.GraphicsFormat.R8G8B8A8_UNorm);
    76.             renderTexture.Create();
    77.  
    78.             frame_width = renderTexture.width;
    79.             frame_height = renderTexture.height;
    80.  
    81.             frame = new Rect(0, 0, frame_width, frame_height);
    82.  
    83.             mainCameraTexture = new Texture2D(frame_width, frame_height, TextureFormat.RGBA32, false);
    84.  
    85.             header = new HeaderMsg();
    86.  
    87.             img_msg = new ImageMsg();
    88.  
    89.             img_msg.width = (uint) frame_width;
    90.             img_msg.height = (uint) frame_height;
    91.             img_msg.step = image_step * (uint) frame_width;
    92.             img_msg.encoding = "rgba8";
    93.  
    94.             infoCamera = CameraInfoGenerator.ConstructCameraInfoMessage(target_camera, header);
    95.  
    96.         }
    97.         else
    98.         {
    99.             Debug.Log("No camera found.");
    100.         }
    101.     }
    102.  
    103.     private void Update()
    104.     {
    105.         if (Camera.main)
    106.         {
    107.             timeElapsed += Time.deltaTime;
    108.  
    109.             if (timeElapsed > (1 / pubMsgFrequency))
    110.             {
    111.                 header.stamp = clock._time;
    112.                 infoCamera.header = header;
    113.  
    114.                 img_msg.header = header;
    115.                 img_msg.data = get_frame_raw();
    116.            
    117.                 ros.Send(imageTopic, img_msg);
    118.                 ros.Send(camInfoTopic, infoCamera);
    119.  
    120.                 timeElapsed = 0;
    121.             }
    122.         }
    123.         else
    124.         {
    125.             Debug.Log("No camera found.");
    126.         }
    127.  
    128.     }
    129.  
    130.     private byte[] get_frame_raw()
    131.     {      
    132.         Camera.main.targetTexture = renderTexture;
    133.         lastTexture = RenderTexture.active;
    134.  
    135.         RenderTexture.active = renderTexture;
    136.         Camera.main.Render();
    137.  
    138.         mainCameraTexture.ReadPixels(frame, 0, 0);
    139.         mainCameraTexture.Apply();
    140.  
    141.         Camera.main.targetTexture = lastTexture;
    142.  
    143.         Camera.main.targetTexture = null;
    144.  
    145.         return mainCameraTexture.GetRawTextureData();;
    146.     }
    147. }
    148.  
     

    Attached Files:

  2. mrpropellers

    mrpropellers

    Unity Technologies

    Joined:
    Jul 10, 2019
    Posts:
    13
    It's hard to say for sure why exactly it's being flipped without knowing more specifics about your platform, but it's likely due to the coordinate convention of the render texture being different than what RViz expects. This is a pretty common issue due to the differences between OpenGL and Direct3D; more on that here. The brute force fix would be to simply copy all the pixels into a new array, in the correct order, before packing them into your Image message. A more optimal solution would be to write a simple shader like the one in the Unity documentation which flips the image vertically, and then use a Graphics.Blit command to write the flipped texture into a temporary texture that you can read into your message. The latter option would require a bit of research and testing to figure out the exact code necessary to work with the version of our rendering pipelines that you're using, but doing it in a shader would be much faster than manually copying the pixels around off-GPU.
     
    MrMagicMoz likes this.
  3. MrMagicMoz

    MrMagicMoz

    Joined:
    Feb 7, 2021
    Posts:
    2
    Thanks for your help, the Shader option sounds interesting and I'll be sure to take a look at it. I did briefly try correcting it with some array operations, which resulted in very low FPS so I don't think that's feasible.
     
  4. mrpropellers

    mrpropellers

    Unity Technologies

    Joined:
    Jul 10, 2019
    Posts:
    13
    Yep manually copying an image pixel by pixel is a costly process - I'd expect it to take on the order of 10-100ms to complete on a 1080p image. This is a pretty common issue though, I've even been bit by it on personal projects before, so there is a lot of discussion and suggestions about it elsewhere on the Unity Forums. These are the most useful ones I found from a quick search:
    https://forum.unity.com/threads/custom-srp-how-to-flip-outputs-for-render-texture.965522/
    https://forum.unity.com/threads/graphics-blit-flips-texture.328288/

    You may also want to follow a pattern closer to what's in the example from the docs for ReadPixels and do your read
    OnPostRender
    . Note that you should be able to just do something like
    Code (CSharp):
    1. renderTexture.ReadTexture(...);
    2. renderTexture.Apply();
    3. var pixels = renderTexture.GetRawTextureData();
    without any of that extra shuffling of active texture references around. If
    RenderTexture.active
    is null, it should simply read from the main camera backbuffer (screen).
     
    aparajithsairam likes this.
  5. maciejw94

    maciejw94

    Joined:
    Feb 11, 2022
    Posts:
    6
    Hi,
    I used your script to get the image from unity to ros. However, the image shown in rqt misses a bunch of elements (basically it shows only a fracture of objects in the scene (pictures attached). They are visible in the preview in Unity but not rqt. Do you know what may be the issue?
    The same thing happens regardless of which camera and camera position I would use. I also thought that it may be the issue of layers so I tweaked the layers of the objects but it is not that.
    Thanks for any tips.
    Cheers
    Maciej
     

    Attached Files:

  6. maciejw94

    maciejw94

    Joined:
    Feb 11, 2022
    Posts:
    6

    following the code, changing the depth of the rendering texture to 8 solves everything :)