Search Unity

  1. Welcome to the Unity Forums! Please take the time to read our Code of Conduct to familiarize yourself with the forum rules and how to post constructively.

Camera.RenderToCubemap including orientation

Discussion in 'General Graphics' started by SCS_Unity, Jun 4, 2018.

  1. SCS_Unity

    SCS_Unity

    Joined:
    Jun 28, 2017
    Posts:
    12
    I'm playing with the new FrameRecorder tool to render a 360° video. During part of the video, my camera rotates on the Z-axis. However, Camera.RenderToCubemap() has no orientation settings, and the outputted image is straight. Is there a way to render these at the correct orientation?
     
    AldeRoberge likes this.
  2. SCS_Unity

    SCS_Unity

    Joined:
    Jun 28, 2017
    Posts:
    12
    Inevitably, someone will google search this, i'll give an answer to my own question. Note: i'm not going to touch stereo rendering for this but with a few alterations to the below you can make this work.

    You have to solve this in 3 steps.
    1) Create a third Render Texture in Camera360Input.cs. It's not a cubemap - it's just a texture. It can be enormous. The dimensions are not square, this will muck with the camera's aspect ratio. Instead, you need the map size's Y to be 3/4 the size of the width because we're going to render it as a cross-cubemap pattern (3 tall, 4 wide)

    Code (CSharp):
    1.             m_Cubemap3 = new RenderTexture(settings360.m_MapSize, settings360.m_MapSize  / 4 * 3, 24, RenderTextureFormat.ARGB32)
    2.             {
    3.                 dimension = TextureDimension.Tex2D
    4.             };

    2) Instead of targetCamera.RenderToCubemap, you need to render a t-cross style cubemap at the 6 angles. This is done in NewFrameReady().

    Code (CSharp):
    1.  
    2. float curFOV = targetCamera.fieldOfView;
    3. Quaternion cameraOrientation = targetCamera.transform.rotation;
    4. RenderTexture currentTex = targetCamera.targetTexture; // save for later
    5.  
    6. targetCamera.fieldOfView = 90; // cubemap rendering happens at 90 degrees
    7. targetCamera.targetTexture = m_Cubemap3; // set the render texture
    8.  
    9. // Render front (+X)              
    10. float py = 1.0f / m_Cubemap3.height; // this is to correct in some off-by-one pixel issues coming up
    11. targetCamera.rect = new Rect(.5f, 1/3.0f - py/2, .25f, 1/3.0f + py);
    12. targetCamera.transform.Rotate(0, 90, 0);
    13. targetCamera.Render();
    14.  
    15. // render right (-Z)
    16. targetCamera.rect = new Rect(.75f, 1/3.0f - py/2, .25f, 1/3.0f + py);
    17. targetCamera.transform.Rotate(0, 90, 0);
    18. targetCamera.Render();
    19.  
    20. // Render back (-X)
    21. targetCamera.rect = new Rect(0, 1/3.0f - py/2, .25f, 1 / 3.0f  + py);
    22. targetCamera.transform.Rotate(0, 90, 0);
    23. targetCamera.Render();
    24.  
    25.  // Render left (+Z)
    26. targetCamera.rect = new Rect(.25f, 1 / 3.0f - py/2, .25f, 1 / 3.0f + py);
    27. targetCamera.transform.Rotate(0, 90, 0);
    28. targetCamera.Render();
    29.  
    30. // render +Y  (up) (left then up)
    31. targetCamera.rect = new Rect(.25f - py/2, 0 - py/2, .25f + py, 1 / 3.0f + py);
    32. targetCamera.transform.Rotate(90, 0, 0);
    33. targetCamera.Render();
    34.  
    35.  // render -Y (down) (left then down)
    36. targetCamera.rect = new Rect(.25f - py/2, 2 / 3.0f - py/2, .25f + py, 1 / 3.0f + py);
    37. targetCamera.transform.Rotate(180, 0, 0);
    38. targetCamera.Render();
    39.  
    40.  
    41. // Reset this camera
    42. targetCamera.rect = new Rect(0, 0, 1, 1);
    43. targetCamera.fieldOfView = curFOV;
    44. targetCamera.transform.rotation = cameraOrientation;
    45. targetCamera.targetTexture = currentTex;
    46.  
    47. // the OLD way
    48. // targetCamera.RenderToCubemap(m_Cubemap1, 63, Camera.MonoOrStereoscopicEye.Mono);
    Ok, now you will render the camera to your own texture as a perfect layout for the next step, which will be to generate the equirectangular texture.

    3) Instead of calling cubemap.ConvertToEquirect(), we're going to use Graphics.Blit() with a Shader/Material that does this for us instead.
    Code (CSharp):
    1. Shader "Unlit/Equirect"
    2. {
    3.     Properties
    4.     {
    5.         _MainTex ("Texture", 2D) = "white" {}
    6.     }
    7.     SubShader
    8.     {
    9.         Tags { "RenderType"="Opaque" "PreviewType"="Plane"}
    10.         LOD 100
    11.  
    12.         Pass
    13.         {
    14.             CGPROGRAM
    15.             #pragma vertex vert
    16.             #pragma fragment frag
    17.             // make fog work
    18.             #pragma multi_compile_fog
    19.          
    20.             #include "UnityCG.cginc"
    21.  
    22.             struct appdata
    23.             {
    24.                 float4 vertex : POSITION;
    25.                 float2 uv : TEXCOORD0;
    26.             };
    27.  
    28.             struct v2f
    29.             {
    30.                 float2 uv : TEXCOORD0;
    31.                 UNITY_FOG_COORDS(1)
    32.                 float4 vertex : SV_POSITION;
    33.             };
    34.  
    35.             sampler2D _MainTex;
    36.             float4 _MainTex_ST;
    37.          
    38.             v2f vert (appdata v)
    39.             {
    40.                 v2f o;
    41.                 o.vertex = UnityObjectToClipPos(v.vertex);
    42.                 o.uv = v.uv;
    43.              
    44.                 return o;
    45.             }
    46.  
    47.             float M_PI = 3.141592654f;
    48.          
    49.             fixed4 frag (v2f i) : SV_Target
    50.             {
    51.              
    52.              
    53.  
    54.                 float theta = (i.uv.x*2 + 1) * 3.141592654;
    55.                 float phi = (-(i.uv.y * 2 - 1) * 3.141592654) / 2;
    56.  
    57.              
    58.  
    59.                 float x = cos(phi) * sin(theta);
    60.                 float y = sin(phi);
    61.                 float z = cos(phi) * cos(theta);
    62.  
    63.                 float scale;
    64.                 float2 px;
    65.                 float2 offset;
    66.              
    67.  
    68.                 // Pull from each of the 6 textures based on the phi/theta pair
    69.                 if (abs(x) >= abs(y) && abs(x) >= abs(z))
    70.                 {              
    71.                     if (x < 0)
    72.                     {
    73.                         scale = -1.0 / x;
    74.                         px.x = (z * scale + 1.0) / 2.0f;
    75.                         px.y = (y * scale + 1.0) / 2.0f;
    76.  
    77.                         // Left
    78.                         return tex2D(_MainTex, float2(px.x / 4, 1 - ((px.y + 1) / 3 )));
    79.                     }
    80.                     else
    81.                     {
    82.                         scale = 1.0 / x;
    83.                         px.x = (-z * scale + 1.0) / 2.0f;
    84.                         px.y = (y * scale + 1.0) / 2.0f;
    85.                         offset.x = 2;
    86.                         offset.y = 1;
    87.  
    88.                         // Right
    89.                         return tex2D(_MainTex, float2(px.x / 4 + .5f, 1 - ((px.y + 1) / 3)));
    90.                     }
    91.                 }
    92.                 else if (abs(y) >= abs(z))
    93.                 {
    94.                     if (y < 0)
    95.                     {
    96.                         scale = -1.0f / y;
    97.                         px.x = (x * scale + 1.0) / 2.0f;
    98.                         px.y = (z * scale + 1.0) / 2.0f;
    99.  
    100.                         // Top
    101.                         return tex2D(_MainTex, float2(px.x / 4 + .25f, 1 - (px.y / 3)));
    102.                     }
    103.                     else
    104.                     {
    105.                         scale = 1.0 / y;
    106.                         px.x = (x * scale + 1.0) / 2.0f;
    107.                         px.y =1 - (-z * scale + 1.0) / 2.0f;
    108.                      
    109.                         // Bottom
    110.                         return tex2D(_MainTex, float2(px.x / 4 + .25f, px.y / 3));
    111.                      
    112.  
    113.                     }
    114.                 }
    115.                 else
    116.                 {
    117.                     if (z < 0)
    118.                     {
    119.                         scale = -1.0 / z;
    120.                         px.x = (-x * scale + 1.0) / 2.0f;
    121.                         px.y = (y * scale + 1.0) / 2.0f;
    122.  
    123.                         // Back
    124.                         return tex2D(_MainTex, float2(px.x / 4 + .75f, 1 - ((px.y + 1) / 3 )));
    125.                     }
    126.                     else
    127.                     {
    128.                         scale = 1.0 / z;
    129.                         px.x = (x * scale + 1.0) / 2.0f;
    130.                         px.y = (y * scale + 1.0) / 2.0f;
    131.  
    132.                         // Front
    133.                         return tex2D(_MainTex, float2(px.x / 4 + .25f, 1 - ((px.y + 1) / 3)));
    134.                     }
    135.                 }
    136.  
    137.                 // Should never come here
    138.                 return float4(1, 0, 1, 1);
    139.             }
    140.             ENDCG
    141.         }
    142.     }
    143. }
    144.  
    Lastly, hook up this shader to the actual Blit() in Camera360Input.cs

    Somewhere, do this to create/save a material:
    Code (CSharp):
    1. m_equirectMat = new Material(Shader.Find("Unlit/Equirect"));
    and then instead of Cubemap.ConvertToEquirect(), do this:
    Code (CSharp):
    1. m_equirectMat.mainTexture = m_Cubemap3;
    2. Graphics.Blit(m_Cubemap3, outputRT, m_equirectMat);
    Works like a charm, and now you can render 360 videos and make your users sick by messing with the orientation at runtime! How fun!
    *Edit for format!
     
  3. Hypertectonic

    Hypertectonic

    Joined:
    Dec 16, 2016
    Posts:
    75
    Thank you for this. I was able to integrate your solution into a script I was working on. But then as I finished and was testing I learned that apparently RenderToCubemap() does in fact follow the camera orientation if it is capturing Stereoscopic Eyes. So to capture a Mono view which follows the camera you just need to do:
    Code (CSharp):
    1. cam.stereoSeparation = 0f;
    2. cam.RenderToCubemap(cubemapMono, 63, Camera.MonoOrStereoscopicEye.Left); //or right, its the same
    And then
    Code (CSharp):
    1. cubemapMono.ConvertToEquirect(equirect, Camera.MonoOrStereoscopicEye.Mono);
    At least it works in 2018.3.10...
     
    Last edited: Jan 13, 2020
  4. Hypertectonic

    Hypertectonic

    Joined:
    Dec 16, 2016
    Posts:
    75
    If anyone was wondering, here's a script that helps with this. It allows one to capture the 360 content out into a render texture, either mono or stereoscopic, following a given camera transform, and save it as PNG or JPG. Haven't finished making it capture a sequence for video yet, but there is a toggle to capture continuously to a render texture.

    Code (CSharp):
    1. //==========================================================
    2. // Purpose: To capture 360 images and videos on runtime.
    3. //
    4. // Usage: Attach to a gameobject with a camera and configure in editor.
    5. //==========================================================
    6.  
    7. using System.IO;
    8. using UnityEngine;
    9. using UnityEngine.Rendering;
    10.  
    11.     public class Hyper_Capture360 : MonoBehaviour
    12.     {
    13.         private RenderTexture cubemapLeftEye;
    14.         private RenderTexture cubemapRightEye;
    15.         private RenderTexture cubemapMono;
    16.         private RenderTexture crossCubemap;
    17.         private RenderTexture equirect_mono;
    18.         //private RenderTexture equirect_stereo;
    19.  
    20.         public string saveFolder = "360_Captures";
    21.         public string fileName = "Capture_01";
    22.         private string fullFileName = "";
    23.         [Tooltip("720, 1080, 1440 (QHD), 2160 (4K)")]  public int verticalResolution = 1080;
    24.         public enum SaveFormat { PNG, JPG };
    25.         public SaveFormat saveFormat = SaveFormat.JPG;
    26.         public enum RenderType { Stereo, Mono, Both };
    27.         public RenderType renderType = RenderType.Mono;
    28.         public bool followCam = true;
    29.         public bool saveEquirectangular = true;
    30.         public bool continuousCapture = false;
    31.  
    32.         public float stereoSeparation = 0.064f;
    33.         [SerializeField] KeyCode captureKey = KeyCode.C;
    34.  
    35.         Camera cam;
    36.  
    37.         void Start()
    38.         {
    39.             string m_Path = Application.dataPath;
    40.  
    41.             //Output the Game data path to the console
    42.             //Debug.Log("Path : " + m_Path);
    43.  
    44.             cubemapLeftEye = new RenderTexture(verticalResolution, verticalResolution, 24, RenderTextureFormat.ARGB32);
    45.             cubemapLeftEye.dimension = TextureDimension.Cube;
    46.             cubemapRightEye = new RenderTexture(verticalResolution, verticalResolution, 24, RenderTextureFormat.ARGB32);
    47.             cubemapRightEye.dimension = TextureDimension.Cube;
    48.             cubemapMono = new RenderTexture(verticalResolution, verticalResolution, 24, RenderTextureFormat.ARGB32);
    49.             cubemapMono.dimension = TextureDimension.Cube;
    50.             crossCubemap = new RenderTexture(verticalResolution, verticalResolution / 4 * 3, 24, RenderTextureFormat.ARGB32) { dimension = TextureDimension.Tex2D };
    51.  
    52.             switch (renderType)
    53.             {
    54.                 case RenderType.Mono:
    55.                     equirect_mono = new RenderTexture(verticalResolution * 2, verticalResolution, 24, RenderTextureFormat.ARGB32); //equirect width should be twice the height of cubemap
    56.                     break;
    57.  
    58.                 case RenderType.Stereo:
    59.                     equirect_mono = new RenderTexture(verticalResolution * 2, verticalResolution * 2, 24, RenderTextureFormat.ARGB32); //equirect height is twice because over-under
    60.                     break;
    61.             }
    62.         }
    63.  
    64.         private void Update()
    65.         {
    66.             // for testing only
    67.             if (Input.GetKeyUp(captureKey))
    68.             {
    69.                 saveEquirectangular = true;
    70.                 Capture360();
    71.                 SaveAsImage();
    72.             }
    73.         }
    74.  
    75.         private void LateUpdate()
    76.         {
    77.             if (continuousCapture)
    78.             {
    79.                 Capture360();
    80.             }
    81.         }
    82.  
    83.         public void Capture360()
    84.         {
    85.             CheckForCamera();
    86.             CaptureCubemaps();
    87.             //if (equirect == null) // why??
    88.             //    return;
    89.             if (saveEquirectangular)
    90.             {
    91.                 ConvertCubeToEquirect();
    92.             }
    93.         }
    94.  
    95.         private void CheckForCamera()
    96.         {
    97.             cam = GetComponent<Camera>();
    98.             if (cam == null)
    99.             {
    100.                 cam = GetComponentInParent<Camera>();
    101.             }
    102.             if (cam == null)
    103.             {
    104.                 Debug.LogWarning("Stereo 360 capture node has no camera or parent camera");
    105.             }
    106.         }
    107.  
    108.         private void CaptureCubemaps()
    109.         {
    110.             switch (renderType)
    111.             {
    112.                 case RenderType.Mono:
    113.                     if (followCam)
    114.                     {
    115.                         cam.stereoSeparation = 0f;
    116.                         cam.RenderToCubemap(cubemapMono, 63, Camera.MonoOrStereoscopicEye.Left); //because stereo rendering does follow camera
    117.                     }
    118.                     else
    119.                     {
    120.                         cam.RenderToCubemap(cubemapMono, 63, Camera.MonoOrStereoscopicEye.Mono);
    121.                     }
    122.                     break;
    123.  
    124.                 case RenderType.Stereo:
    125.                     cam.stereoSeparation = stereoSeparation;
    126.                     cam.RenderToCubemap(cubemapLeftEye, 63, Camera.MonoOrStereoscopicEye.Left);
    127.                     cam.RenderToCubemap(cubemapRightEye, 63, Camera.MonoOrStereoscopicEye.Right);
    128.                     if (!followCam)
    129.                     {
    130.                         Debug.LogWarning("Follow Cam is always enabled for Stereo Rendering");
    131.                     }
    132.  
    133.                     break;
    134.             }
    135.         }
    136.  
    137.         private void ConvertCubeToEquirect()
    138.         {
    139.             switch (renderType)
    140.             {
    141.                 case RenderType.Mono:
    142.                     equirect_mono = new RenderTexture(verticalResolution * 2, verticalResolution, 24, RenderTextureFormat.ARGB32); //equirect width should be twice the height of cubemap
    143.                     if (followCam)
    144.                     {
    145.                         cubemapMono.ConvertToEquirect(equirect_mono, Camera.MonoOrStereoscopicEye.Mono);
    146.                     }
    147.                     else
    148.                     {
    149.                         cubemapMono.ConvertToEquirect(equirect_mono, Camera.MonoOrStereoscopicEye.Mono);
    150.                     }
    151.                     break;
    152.  
    153.                 case RenderType.Stereo:
    154.                     equirect_mono = new RenderTexture(verticalResolution * 2, verticalResolution * 2, 24, RenderTextureFormat.ARGB32); //equirect height is twice because over-under
    155.                     cubemapLeftEye.ConvertToEquirect(equirect_mono, Camera.MonoOrStereoscopicEye.Left);
    156.                     cubemapRightEye.ConvertToEquirect(equirect_mono, Camera.MonoOrStereoscopicEye.Right);
    157.                     break;
    158.             }
    159.         }
    160.  
    161.         private void SaveAsImage()
    162.         {
    163.             if (renderType == RenderType.Mono)
    164.             {
    165.                 fullFileName = (Application.dataPath + Path.DirectorySeparatorChar + saveFolder + Path.DirectorySeparatorChar + fileName);
    166.             }
    167.             else
    168.             {
    169.                 fullFileName = (Application.dataPath + Path.DirectorySeparatorChar + saveFolder + Path.DirectorySeparatorChar + fileName + "_stereo");
    170.             }
    171.  
    172.             if (saveFormat == SaveFormat.PNG)
    173.             {
    174.                 SavePNG(equirect_mono, fullFileName);
    175.             }
    176.             else
    177.             {
    178.                 SaveJPG(equirect_mono, fullFileName);
    179.             }
    180.         }
    181.  
    182.         public static void SavePNG(RenderTexture rt, string filePath)
    183.         {
    184.             byte[] bytes = ToTexture2D(rt).EncodeToPNG();
    185.             File.WriteAllBytes(filePath + ".png", bytes);
    186.             //Debug.Log("Saved");
    187.         }
    188.  
    189.         public static void SaveJPG(RenderTexture rt, string filePath, int quality = 75)
    190.         {
    191.             byte[] bytes = ToTexture2D(rt).EncodeToJPG(quality);
    192.             File.WriteAllBytes(filePath + ".jpg", bytes);
    193.             //Debug.Log("Saved");
    194.         }
    195.  
    196.         private static Texture2D ToTexture2D(RenderTexture rTex)
    197.         {
    198.             Texture2D tex = new Texture2D(rTex.width, rTex.height, TextureFormat.RGB24, false);
    199.             RenderTexture.active = rTex;
    200.             tex.ReadPixels(new Rect(0, 0, rTex.width, rTex.height), 0, 0);
    201.             tex.Apply();
    202.             return tex;
    203.         }
    204.     }
    205.  
     
  5. Sbarn

    Sbarn

    Joined:
    Jan 25, 2017
    Posts:
    2
    Hi, sorry for this request but where this code go? in Camera360Input.cs?
     
  6. olli_vrcoaster

    olli_vrcoaster

    Joined:
    Sep 1, 2017
    Posts:
    24
    thanks, but weird that this is not documented. Why not give this function a seperate parameter like (bool useOrientation)
     
    matsuda-san and MUGIK like this.