Camera.RenderToCubemap including orientation

I'm playing with the new FrameRecorder tool to render a 360° video. During part of the video, my camera rotates on the Z-axis. However, Camera.RenderToCubemap() has no orientation settings, and the outputted image is straight. Is there a way to render these at the correct orientation?

 

Inevitably, someone will google search this, i'll give an answer to my own question. Note: i'm not going to touch stereo rendering for this but with a few alterations to the below you can make this work.

You have to solve this in 3 steps.
1) Create a third Render Texture in Camera360Input.cs. It's not a cubemap - it's just a texture. It can be enormous. The dimensions are not square, this will muck with the camera's aspect ratio. Instead, you need the map size's Y to be 3/4 the size of the width because we're going to render it as a cross-cubemap pattern (3 tall, 4 wide)

Code (CSharp):

1.             m_Cubemap3 = new RenderTexture(settings360.m_MapSize, settings360.m_MapSize  / 4 * 3, 24, RenderTextureFormat.ARGB32)
2.             {
3.                 dimension = TextureDimension.Tex2D
4.             };

2) Instead of targetCamera.RenderToCubemap, you need to render a t-cross style cubemap at the 6 angles. This is done in NewFrameReady().

Code (CSharp):

1.  
2. float curFOV = targetCamera.fieldOfView;
3. Quaternion cameraOrientation = targetCamera.transform.rotation;
4. RenderTexture currentTex = targetCamera.targetTexture; // save for later
5.  
6. targetCamera.fieldOfView = 90; // cubemap rendering happens at 90 degrees
7. targetCamera.targetTexture = m_Cubemap3; // set the render texture
8.  
9. // Render front (+X)              
10. float py = 1.0f / m_Cubemap3.height; // this is to correct in some off-by-one pixel issues coming up
11. targetCamera.rect = new Rect(.5f, 1/3.0f - py/2, .25f, 1/3.0f + py);
12. targetCamera.transform.Rotate(0, 90, 0);
13. targetCamera.Render();
14.  
15. // render right (-Z)
16. targetCamera.rect = new Rect(.75f, 1/3.0f - py/2, .25f, 1/3.0f + py);
17. targetCamera.transform.Rotate(0, 90, 0);
18. targetCamera.Render();
19.  
20. // Render back (-X)
21. targetCamera.rect = new Rect(0, 1/3.0f - py/2, .25f, 1 / 3.0f  + py);
22. targetCamera.transform.Rotate(0, 90, 0);
23. targetCamera.Render();
24.  
25.  // Render left (+Z)
26. targetCamera.rect = new Rect(.25f, 1 / 3.0f - py/2, .25f, 1 / 3.0f + py);
27. targetCamera.transform.Rotate(0, 90, 0);
28. targetCamera.Render();
29.  
30. // render +Y  (up) (left then up)
31. targetCamera.rect = new Rect(.25f - py/2, 0 - py/2, .25f + py, 1 / 3.0f + py);
32. targetCamera.transform.Rotate(90, 0, 0);
33. targetCamera.Render();
34.  
35.  // render -Y (down) (left then down)
36. targetCamera.rect = new Rect(.25f - py/2, 2 / 3.0f - py/2, .25f + py, 1 / 3.0f + py);
37. targetCamera.transform.Rotate(180, 0, 0);
38. targetCamera.Render();
39.  
40.  
41. // Reset this camera
42. targetCamera.rect = new Rect(0, 0, 1, 1);
43. targetCamera.fieldOfView = curFOV;
44. targetCamera.transform.rotation = cameraOrientation;
45. targetCamera.targetTexture = currentTex;
46.  
47. // the OLD way
48. // targetCamera.RenderToCubemap(m_Cubemap1, 63, Camera.MonoOrStereoscopicEye.Mono);

Ok, now you will render the camera to your own texture as a perfect layout for the next step, which will be to generate the equirectangular texture.

3) Instead of calling cubemap.ConvertToEquirect(), we're going to use Graphics.Blit() with a Shader/Material that does this for us instead.

Code (CSharp):

1. Shader "Unlit/Equirect"
2. {
3.     Properties
4.     {
5.         _MainTex ("Texture", 2D) = "white" {}
6.     }
7.     SubShader
8.     {
9.         Tags { "RenderType"="Opaque" "PreviewType"="Plane"}
10.         LOD 100
11.  
12.         Pass
13.         {
14.             CGPROGRAM
15.             #pragma vertex vert
16.             #pragma fragment frag
17.             // make fog work
18.             #pragma multi_compile_fog
19.          
20.             #include "UnityCG.cginc"
21.  
22.             struct appdata
23.             {
24.                 float4 vertex : POSITION;
25.                 float2 uv : TEXCOORD0;
26.             };
27.  
28.             struct v2f
29.             {
30.                 float2 uv : TEXCOORD0;
31.                 UNITY_FOG_COORDS(1)
32.                 float4 vertex : SV_POSITION;
33.             };
34.  
35.             sampler2D _MainTex;
36.             float4 _MainTex_ST;
37.          
38.             v2f vert (appdata v)
39.             {
40.                 v2f o;
41.                 o.vertex = UnityObjectToClipPos(v.vertex);
42.                 o.uv = v.uv;
43.              
44.                 return o;
45.             }
46.  
47.             float M_PI = 3.141592654f;
48.          
49.             fixed4 frag (v2f i) : SV_Target
50.             {
51.              
52.              
53.  
54.                 float theta = (i.uv.x*2 + 1) * 3.141592654;
55.                 float phi = (-(i.uv.y * 2 - 1) * 3.141592654) / 2;
56.  
57.              
58.  
59.                 float x = cos(phi) * sin(theta);
60.                 float y = sin(phi);
61.                 float z = cos(phi) * cos(theta);
62.  
63.                 float scale;
64.                 float2 px;
65.                 float2 offset;
66.              
67.  
68.                 // Pull from each of the 6 textures based on the phi/theta pair
69.                 if (abs(x) >= abs(y) && abs(x) >= abs(z))
70.                 {              
71.                     if (x < 0)
72.                     {
73.                         scale = -1.0 / x;
74.                         px.x = (z * scale + 1.0) / 2.0f;
75.                         px.y = (y * scale + 1.0) / 2.0f;
76.  
77.                         // Left
78.                         return tex2D(_MainTex, float2(px.x / 4, 1 - ((px.y + 1) / 3 )));
79.                     }
80.                     else
81.                     {
82.                         scale = 1.0 / x;
83.                         px.x = (-z * scale + 1.0) / 2.0f;
84.                         px.y = (y * scale + 1.0) / 2.0f;
85.                         offset.x = 2;
86.                         offset.y = 1;
87.  
88.                         // Right
89.                         return tex2D(_MainTex, float2(px.x / 4 + .5f, 1 - ((px.y + 1) / 3)));
90.                     }
91.                 }
92.                 else if (abs(y) >= abs(z))
93.                 {
94.                     if (y < 0)
95.                     {
96.                         scale = -1.0f / y;
97.                         px.x = (x * scale + 1.0) / 2.0f;
98.                         px.y = (z * scale + 1.0) / 2.0f;
99.  
100.                         // Top
101.                         return tex2D(_MainTex, float2(px.x / 4 + .25f, 1 - (px.y / 3)));
102.                     }
103.                     else
104.                     {
105.                         scale = 1.0 / y;
106.                         px.x = (x * scale + 1.0) / 2.0f;
107.                         px.y =1 - (-z * scale + 1.0) / 2.0f;
108.                      
109.                         // Bottom
110.                         return tex2D(_MainTex, float2(px.x / 4 + .25f, px.y / 3));
111.                      
112.  
113.                     }
114.                 }
115.                 else
116.                 {
117.                     if (z < 0)
118.                     {
119.                         scale = -1.0 / z;
120.                         px.x = (-x * scale + 1.0) / 2.0f;
121.                         px.y = (y * scale + 1.0) / 2.0f;
122.  
123.                         // Back
124.                         return tex2D(_MainTex, float2(px.x / 4 + .75f, 1 - ((px.y + 1) / 3 )));
125.                     }
126.                     else
127.                     {
128.                         scale = 1.0 / z;
129.                         px.x = (x * scale + 1.0) / 2.0f;
130.                         px.y = (y * scale + 1.0) / 2.0f;
131.  
132.                         // Front
133.                         return tex2D(_MainTex, float2(px.x / 4 + .25f, 1 - ((px.y + 1) / 3)));
134.                     }
135.                 }
136.  
137.                 // Should never come here
138.                 return float4(1, 0, 1, 1);
139.             }
140.             ENDCG
141.         }
142.     }
143. }
144.  

Lastly, hook up this shader to the actual Blit() in Camera360Input.cs

Somewhere, do this to create/save a material:

Code (CSharp):

1. m_equirectMat = new Material(Shader.Find("Unlit/Equirect"));

and then instead of Cubemap.ConvertToEquirect(), do this:

Code (CSharp):

1. m_equirectMat.mainTexture = m_Cubemap3;
2. Graphics.Blit(m_Cubemap3, outputRT, m_equirectMat);

Works like a charm, and now you can render 360 videos and make your users sick by messing with the orientation at runtime! How fun!
*Edit for format!

 

Thank you for this. I was able to integrate your solution into a script I was working on. But then as I finished and was testing I learned that apparently RenderToCubemap() does in fact follow the camera orientation if it is capturing Stereoscopic Eyes. So to capture a Mono view which follows the camera you just need to do:

Code (CSharp):

1. cam.stereoSeparation = 0f;
2. cam.RenderToCubemap(cubemapMono, 63, Camera.MonoOrStereoscopicEye.Left); //or right, its the same

And then

Code (CSharp):

1. cubemapMono.ConvertToEquirect(equirect, Camera.MonoOrStereoscopicEye.Mono);

At least it works in 2018.3.10...

 

If anyone was wondering, here's a script that helps with this. It allows one to capture the 360 content out into a render texture, either mono or stereoscopic, following a given camera transform, and save it as PNG or JPG. Haven't finished making it capture a sequence for video yet, but there is a toggle to capture continuously to a render texture.

Code (CSharp):

1. //==========================================================
2. // Purpose: To capture 360 images and videos on runtime.
3. //
4. // Usage: Attach to a gameobject with a camera and configure in editor.
5. //==========================================================
6.  
7. using System.IO;
8. using UnityEngine;
9. using UnityEngine.Rendering;
10.  
11.     public class Hyper_Capture360 : MonoBehaviour
12.     {
13.         private RenderTexture cubemapLeftEye;
14.         private RenderTexture cubemapRightEye;
15.         private RenderTexture cubemapMono;
16.         private RenderTexture crossCubemap;
17.         private RenderTexture equirect_mono;
18.         //private RenderTexture equirect_stereo;
19.  
20.         public string saveFolder = "360_Captures";
21.         public string fileName = "Capture_01";
22.         private string fullFileName = "";
23.         [Tooltip("720, 1080, 1440 (QHD), 2160 (4K)")]  public int verticalResolution = 1080;
24.         public enum SaveFormat { PNG, JPG };
25.         public SaveFormat saveFormat = SaveFormat.JPG;
26.         public enum RenderType { Stereo, Mono, Both };
27.         public RenderType renderType = RenderType.Mono;
28.         public bool followCam = true;
29.         public bool saveEquirectangular = true;
30.         public bool continuousCapture = false;
31.  
32.         public float stereoSeparation = 0.064f;
33.         [SerializeField] KeyCode captureKey = KeyCode.C;
34.  
35.         Camera cam;
36.  
37.         void Start()
38.         {
39.             string m_Path = Application.dataPath;
40.  
41.             //Output the Game data path to the console
42.             //Debug.Log("Path : " + m_Path);
43.  
44.             cubemapLeftEye = new RenderTexture(verticalResolution, verticalResolution, 24, RenderTextureFormat.ARGB32);
45.             cubemapLeftEye.dimension = TextureDimension.Cube;
46.             cubemapRightEye = new RenderTexture(verticalResolution, verticalResolution, 24, RenderTextureFormat.ARGB32);
47.             cubemapRightEye.dimension = TextureDimension.Cube;
48.             cubemapMono = new RenderTexture(verticalResolution, verticalResolution, 24, RenderTextureFormat.ARGB32);
49.             cubemapMono.dimension = TextureDimension.Cube;
50.             crossCubemap = new RenderTexture(verticalResolution, verticalResolution / 4 * 3, 24, RenderTextureFormat.ARGB32) { dimension = TextureDimension.Tex2D };
51.  
52.             switch (renderType)
53.             {
54.                 case RenderType.Mono:
55.                     equirect_mono = new RenderTexture(verticalResolution * 2, verticalResolution, 24, RenderTextureFormat.ARGB32); //equirect width should be twice the height of cubemap
56.                     break;
57.  
58.                 case RenderType.Stereo:
59.                     equirect_mono = new RenderTexture(verticalResolution * 2, verticalResolution * 2, 24, RenderTextureFormat.ARGB32); //equirect height is twice because over-under
60.                     break;
61.             }
62.         }
63.  
64.         private void Update()
65.         {
66.             // for testing only
67.             if (Input.GetKeyUp(captureKey))
68.             {
69.                 saveEquirectangular = true;
70.                 Capture360();
71.                 SaveAsImage();
72.             }
73.         }
74.  
75.         private void LateUpdate()
76.         {
77.             if (continuousCapture)
78.             {
79.                 Capture360();
80.             }
81.         }
82.  
83.         public void Capture360()
84.         {
85.             CheckForCamera();
86.             CaptureCubemaps();
87.             //if (equirect == null) // why??
88.             //    return;
89.             if (saveEquirectangular)
90.             {
91.                 ConvertCubeToEquirect();
92.             }
93.         }
94.  
95.         private void CheckForCamera()
96.         {
97.             cam = GetComponent<Camera>();
98.             if (cam == null)
99.             {
100.                 cam = GetComponentInParent<Camera>();
101.             }
102.             if (cam == null)
103.             {
104.                 Debug.LogWarning("Stereo 360 capture node has no camera or parent camera");
105.             }
106.         }
107.  
108.         private void CaptureCubemaps()
109.         {
110.             switch (renderType)
111.             {
112.                 case RenderType.Mono:
113.                     if (followCam)
114.                     {
115.                         cam.stereoSeparation = 0f;
116.                         cam.RenderToCubemap(cubemapMono, 63, Camera.MonoOrStereoscopicEye.Left); //because stereo rendering does follow camera
117.                     }
118.                     else
119.                     {
120.                         cam.RenderToCubemap(cubemapMono, 63, Camera.MonoOrStereoscopicEye.Mono);
121.                     }
122.                     break;
123.  
124.                 case RenderType.Stereo:
125.                     cam.stereoSeparation = stereoSeparation;
126.                     cam.RenderToCubemap(cubemapLeftEye, 63, Camera.MonoOrStereoscopicEye.Left);
127.                     cam.RenderToCubemap(cubemapRightEye, 63, Camera.MonoOrStereoscopicEye.Right);
128.                     if (!followCam)
129.                     {
130.                         Debug.LogWarning("Follow Cam is always enabled for Stereo Rendering");
131.                     }
132.  
133.                     break;
134.             }
135.         }
136.  
137.         private void ConvertCubeToEquirect()
138.         {
139.             switch (renderType)
140.             {
141.                 case RenderType.Mono:
142.                     equirect_mono = new RenderTexture(verticalResolution * 2, verticalResolution, 24, RenderTextureFormat.ARGB32); //equirect width should be twice the height of cubemap
143.                     if (followCam)
144.                     {
145.                         cubemapMono.ConvertToEquirect(equirect_mono, Camera.MonoOrStereoscopicEye.Mono);
146.                     }
147.                     else
148.                     {
149.                         cubemapMono.ConvertToEquirect(equirect_mono, Camera.MonoOrStereoscopicEye.Mono);
150.                     }
151.                     break;
152.  
153.                 case RenderType.Stereo:
154.                     equirect_mono = new RenderTexture(verticalResolution * 2, verticalResolution * 2, 24, RenderTextureFormat.ARGB32); //equirect height is twice because over-under
155.                     cubemapLeftEye.ConvertToEquirect(equirect_mono, Camera.MonoOrStereoscopicEye.Left);
156.                     cubemapRightEye.ConvertToEquirect(equirect_mono, Camera.MonoOrStereoscopicEye.Right);
157.                     break;
158.             }
159.         }
160.  
161.         private void SaveAsImage()
162.         {
163.             if (renderType == RenderType.Mono)
164.             {
165.                 fullFileName = (Application.dataPath + Path.DirectorySeparatorChar + saveFolder + Path.DirectorySeparatorChar + fileName);
166.             }
167.             else
168.             {
169.                 fullFileName = (Application.dataPath + Path.DirectorySeparatorChar + saveFolder + Path.DirectorySeparatorChar + fileName + "_stereo");
170.             }
171.  
172.             if (saveFormat == SaveFormat.PNG)
173.             {
174.                 SavePNG(equirect_mono, fullFileName);
175.             }
176.             else
177.             {
178.                 SaveJPG(equirect_mono, fullFileName);
179.             }
180.         }
181.  
182.         public static void SavePNG(RenderTexture rt, string filePath)
183.         {
184.             byte[] bytes = ToTexture2D(rt).EncodeToPNG();
185.             File.WriteAllBytes(filePath + ".png", bytes);
186.             //Debug.Log("Saved");
187.         }
188.  
189.         public static void SaveJPG(RenderTexture rt, string filePath, int quality = 75)
190.         {
191.             byte[] bytes = ToTexture2D(rt).EncodeToJPG(quality);
192.             File.WriteAllBytes(filePath + ".jpg", bytes);
193.             //Debug.Log("Saved");
194.         }
195.  
196.         private static Texture2D ToTexture2D(RenderTexture rTex)
197.         {
198.             Texture2D tex = new Texture2D(rTex.width, rTex.height, TextureFormat.RGB24, false);
199.             RenderTexture.active = rTex;
200.             tex.ReadPixels(new Rect(0, 0, rTex.width, rTex.height), 0, 0);
201.             tex.Apply();
202.             return tex;
203.         }
204.     }
205.  

 

Hi, sorry for this request but where this code go? in Camera360Input.cs?

 

thanks, but weird that this is not documented. Why not give this function a seperate parameter like (bool useOrientation)