GetRawTextureData() Method resulting in flipped image in RViz

I am currently publishing an image to ROS from my unity program, with the hope of tracking an AprilTag in the scene. However when viewing the image in RViz the image is flipped along its x axis and the AprilTag package cannot track the tag unless I flip the tag in the scene. What could be problem here ?

Code (CSharp):

1. using UnityEngine;
2. using Unity.Robotics.ROSTCPConnector;
3. using RosMessageTypes.UnityRoboticsDemo;
4. using RosMessageTypes.Sensor;
5. using RosMessageTypes.Std;
6. using RosMessageTypes.BuiltinInterfaces;
7. using System.Collections;
8.  
9. /// <summary>
10. ///
11. /// </summary>
12.  
13. [RequireComponent(typeof(ROSClockSubscriber))]
14. public class ROSCameraDataPublisher : MonoBehaviour
15. {
16.     ROSConnection ros;
17.     public string imageTopic = "/camera_rect/image_rect";
18.     public string camInfoTopic = "/camera_rect/camera_info";
19.  
20.     public string CompressedImageTopic = "/camera_rect/image_rect_compressed";
21.  
22.     public Camera target_camera;
23.  
24.     public bool compressed = false;
25.  
26.     public float pubMsgFrequency = 30f;
27.  
28.     private float timeElapsed;
29.     private RenderTexture renderTexture;
30.     private RenderTexture lastTexture;
31.  
32.     private Texture2D mainCameraTexture;
33.     private Rect frame;
34.  
35.  
36.     private int frame_width;
37.     private int frame_height;
38.     private const int isBigEndian = 0;
39.     private uint image_step = 4;
40.     TimeMsg lastTime;
41.  
42.     private ROSClockSubscriber clock;
43.  
44.     private ImageMsg img_msg;
45.     private CameraInfoMsg infoCamera;
46.  
47.     private HeaderMsg header;
48.  
49.     void Start()
50.     {
51.         // start the ROS connection
52.         ros = ROSConnection.instance;
53.  
54.         if(ros)
55.         {
56.             ros.RegisterPublisher<ImageMsg>(imageTopic);
57.             ros.RegisterPublisher<CompressedImageMsg>(CompressedImageTopic);
58.  
59.             ros.RegisterPublisher<CameraInfoMsg>(camInfoTopic);
60.             clock = GetComponent<ROSClockSubscriber>();
61.         }
62.         else
63.         {
64.             Debug.Log("No ros connection found.");
65.         }
66.  
67.  
68.         if (!target_camera)
69.         {
70.             target_camera = Camera.main;
71.         }
72.  
73.         if (Camera.main)
74.         {
75.             renderTexture = new RenderTexture(Camera.main.pixelWidth, Camera.main.pixelHeight, 0, UnityEngine.Experimental.Rendering.GraphicsFormat.R8G8B8A8_UNorm);
76.             renderTexture.Create();
77.  
78.             frame_width = renderTexture.width;
79.             frame_height = renderTexture.height;
80.  
81.             frame = new Rect(0, 0, frame_width, frame_height);
82.  
83.             mainCameraTexture = new Texture2D(frame_width, frame_height, TextureFormat.RGBA32, false);
84.  
85.             header = new HeaderMsg();
86.  
87.             img_msg = new ImageMsg();
88.  
89.             img_msg.width = (uint) frame_width;
90.             img_msg.height = (uint) frame_height;
91.             img_msg.step = image_step * (uint) frame_width;
92.             img_msg.encoding = "rgba8";
93.  
94.             infoCamera = CameraInfoGenerator.ConstructCameraInfoMessage(target_camera, header);
95.  
96.         }
97.         else
98.         {
99.             Debug.Log("No camera found.");
100.         }
101.     }
102.  
103.     private void Update()
104.     {
105.         if (Camera.main)
106.         {
107.             timeElapsed += Time.deltaTime;
108.  
109.             if (timeElapsed > (1 / pubMsgFrequency))
110.             {
111.                 header.stamp = clock._time;
112.                 infoCamera.header = header;
113.  
114.                 img_msg.header = header;
115.                 img_msg.data = get_frame_raw();
116.            
117.                 ros.Send(imageTopic, img_msg);
118.                 ros.Send(camInfoTopic, infoCamera);
119.  
120.                 timeElapsed = 0;
121.             }
122.         }
123.         else
124.         {
125.             Debug.Log("No camera found.");
126.         }
127.  
128.     }
129.  
130.     private byte[] get_frame_raw()
131.     {      
132.         Camera.main.targetTexture = renderTexture;
133.         lastTexture = RenderTexture.active;
134.  
135.         RenderTexture.active = renderTexture;
136.         Camera.main.Render();
137.  
138.         mainCameraTexture.ReadPixels(frame, 0, 0);
139.         mainCameraTexture.Apply();
140.  
141.         Camera.main.targetTexture = lastTexture;
142.  
143.         Camera.main.targetTexture = null;
144.  
145.         return mainCameraTexture.GetRawTextureData();;
146.     }
147. }
148.  

 

Thanks for your help, the Shader option sounds interesting and I'll be sure to take a look at it. I did briefly try correcting it with some array operations, which resulted in very low FPS so I don't think that's feasible.

 

Hi,
I used your script to get the image from unity to ros. However, the image shown in rqt misses a bunch of elements (basically it shows only a fracture of objects in the scene (pictures attached). They are visible in the preview in Unity but not rqt. Do you know what may be the issue?
The same thing happens regardless of which camera and camera position I would use. I also thought that it may be the issue of layers so I tweaked the layers of the objects but it is not that.
Thanks for any tips.
Cheers
Maciej

 

following the code, changing the depth of the rendering texture to 8 solves everything