Search Unity

Linear Color Space for Camera Background

Discussion in 'Handheld AR' started by blanx, Jun 22, 2018.

  1. blanx

    blanx

    Joined:
    Feb 26, 2016
    Posts:
    47
    I just started using the foundation framework to see how it works out.

    During testing I wondered how to use a proper linear camera background shader instead of the default one. Seems like the background image is to bright when setting the player settings to linear.

    I tried to plug the arkit camera shader into a material and use this as override, but this gives me black background.

    Is there a sample shader available that will work?
     
  2. SilverBugi

    SilverBugi

    Joined:
    Jan 30, 2017
    Posts:
    4
    Is there any solution for this problem? I'm having it too!
     
  3. Krxtopher

    Krxtopher

    Joined:
    Nov 30, 2013
    Posts:
    3
    I'm using AR Foundation version 1.0.0-preview.17 and ARKit XR Plugin version 1.0.0-preview.13. Here's the solution I'm using.

    Create a new shader by coping the shader code from
    Packages/ARKit XR Plugin/Runtime/iOS/Resource/ARKitShader
    .

    Change this line (near the end of the file):

    Code (CSharp):
    1. return mul(ycbcrToRGBTransform, ycbcr);
    ...to this...

    Code (CSharp):
    1. return pow(mul(ycbcrToRGBTransform, ycbcr), 2.2);
    Then create a new material that uses that shader and assign that material to your AR Camera's "AR Camera Background" component's "Custom Material" property.
     
  4. waldgeist

    waldgeist

    Joined:
    May 6, 2017
    Posts:
    48
    @Krxtopher Thanks for that hint. But how would I do this if I have to support both Android and iOS?

    (I am looking for a way to create my own custom shader to dim the background video programmatically. If there's any other way to do this, I would be happy to know about it.)
     
  5. tdmowrer

    tdmowrer

    Unity Technologies

    Joined:
    Apr 21, 2017
    Posts:
    518
  6. axelrosie

    axelrosie

    Joined:
    Nov 13, 2018
    Posts:
    3
    Hey guys, I'm having troubles with that. :)
    I have tried with this solution (https://github.com/Unity-Technologies/arfoundation-samples/issues/46)
    and this other (https://github.com/google-ar/arcore-unity-sdk/issues/268)
    but stills, everytime I launch the android app I get a black screen from the begining, it doesnt even load the Unity splash screen. :/

    I have checked on LogCat and I'm getting this error:

    [EGL] Failed to create window surface: EGL_BAD_MATCH: Arguments are inconsitent (for example, a valid context requires buffers not supplied by a valid surface)

    Do you guys have any idea of what might be it or what I might be doing wrong?

    Thanks
     
  7. axelrosie

    axelrosie

    Joined:
    Nov 13, 2018
    Posts:
    3
    Just in case anyone else is facing this problem, we fixed it enabling on "ProjectSettings/Resolution and Presentation" the checkbox "Use 32-bit Display Buffer" (Unity 2018.2.14)
     
    newguy123 likes this.
  8. GemBot

    GemBot

    Joined:
    May 1, 2014
    Posts:
    1
    Hi all,

    to support linear color space for android and ios you need a script to do the job and two materials with a dedicated shader for ARCore and ARKit (as @tdmowrer suggested).

    you can use this script. attach it to the game object where your ARCameraBackground script resides in your AR scene.

    Code (CSharp):
    1. //
    2. //
    3. // purpose: set camera background material depending on platform used
    4. //
    5. //
    6. // author: ge
    7. //
    8. // (c) 2019 innovation.rocks consulting gmbh
    9. //
    10.  
    11. using UnityEngine;
    12. using UnityEngine.XR.ARFoundation;
    13.  
    14. [RequireComponent(typeof(ARCameraBackground))]
    15. public class ARCameraPlatformBackground : MonoBehaviour
    16. {
    17.     public Material _ARCoreBackground;
    18.     public Material _ARKitBackground;
    19.  
    20.     void Start ()
    21.     {
    22.         ARCameraBackground arcbg = GetComponent<ARCameraBackground> ();
    23.         arcbg.useCustomMaterial = true;
    24. #if UNITY_IOS
    25.         arcbg.customMaterial = _ARKitBackground;
    26. #else
    27.         arcbg.customMaterial = _ARCoreBackground;
    28. #endif
    29.     }
    30. }
    31.  

    make two materials for ARCore and ARKit with the following shaders attached


    ARCore Shader

    Code (CSharp):
    1. Shader "inr/AR/ARCore Background Linear"
    2. {
    3.     Properties {
    4.         _MainTex ("Texture", 2D) = "white" {}
    5.         _UvTopLeftRight ("UV of top corners", Vector) = (0, 1, 1, 1)
    6.         _UvBottomLeftRight ("UV of bottom corners", Vector) = (0 , 0, 1, 0)
    7.         _Gamma("Gamma", Float) = 2.2
    8.     }
    9.  
    10.     // For GLES3
    11.     SubShader
    12.     {
    13.         Pass
    14.         {
    15.             ZWrite Off
    16.  
    17.             GLSLPROGRAM
    18.  
    19.             #pragma only_renderers gles3
    20.  
    21.             #ifdef SHADER_API_GLES3
    22.             #extension GL_OES_EGL_image_external_essl3 : require
    23.             #endif
    24.  
    25.             uniform vec4 _UvTopLeftRight;
    26.             uniform vec4 _UvBottomLeftRight;
    27.  
    28.             #ifdef VERTEX
    29.  
    30.             varying vec2 textureCoord;
    31.  
    32.             void main()
    33.             {
    34.                 #ifdef SHADER_API_GLES3
    35.                 vec2 uvTop = mix(_UvTopLeftRight.xy, _UvTopLeftRight.zw, gl_MultiTexCoord0.x);
    36.                 vec2 uvBottom = mix(_UvBottomLeftRight.xy, _UvBottomLeftRight.zw, gl_MultiTexCoord0.x);
    37.                 textureCoord = mix(uvTop, uvBottom, gl_MultiTexCoord0.y);
    38.  
    39.                 gl_Position = gl_ModelViewProjectionMatrix * gl_Vertex;
    40.                 #endif
    41.             }
    42.  
    43.             #endif
    44.  
    45.             #ifdef FRAGMENT
    46.             varying vec2 textureCoord;
    47.             uniform samplerExternalOES _MainTex;
    48.             uniform float _Gamma;
    49.  
    50.             void main()
    51.             {
    52.                 #ifdef SHADER_API_GLES3
    53.                 vec4 color = texture(_MainTex, textureCoord);
    54.  
    55.                 color.rgb = pow(color.rgb, vec3(_Gamma, _Gamma, _Gamma));
    56.                 color.rgb = clamp(color.rgb, 0.0, 0.996);
    57.  
    58.                 gl_FragColor = color;
    59.                 #endif
    60.             }
    61.  
    62.             #endif
    63.  
    64.             ENDGLSL
    65.         }
    66.     }
    67.  
    68.     Subshader
    69.     {
    70.         Pass
    71.         {
    72.             ZWrite Off
    73.  
    74.             CGPROGRAM
    75.  
    76.             #pragma exclude_renderers gles3
    77.             #pragma vertex vert
    78.             #pragma fragment frag
    79.  
    80.             #include "UnityCG.cginc"
    81.  
    82.             uniform float4 _UvTopLeftRight;
    83.             uniform float4 _UvBottomLeftRight;
    84.             uniform float _Gamma;
    85.  
    86.             struct appdata
    87.             {
    88.                 float4 vertex : POSITION;
    89.                 float2 uv : TEXCOORD0;
    90.             };
    91.  
    92.             struct v2f
    93.             {
    94.                 float2 uv : TEXCOORD0;
    95.                 float4 vertex : SV_POSITION;
    96.             };
    97.  
    98.             v2f vert(appdata v)
    99.             {
    100.                 float2 uvTop = lerp(_UvTopLeftRight.xy, _UvTopLeftRight.zw, v.uv.x);
    101.                 float2 uvBottom = lerp(_UvBottomLeftRight.xy, _UvBottomLeftRight.zw, v.uv.x);
    102.  
    103.                 v2f o;
    104.                 o.vertex = UnityObjectToClipPos(v.vertex);
    105.                 o.uv = lerp(uvTop, uvBottom, v.uv.y);
    106.  
    107.                 // Instant preview's texture is transformed differently.
    108.                 o.uv = o.uv.yx;
    109.                 o.uv.x = 1.0 - o.uv.x;
    110.  
    111.                 return o;
    112.             }
    113.  
    114.             sampler2D _MainTex;
    115.  
    116.             fixed4 frag(v2f i) : SV_Target
    117.             {
    118.                 fixed4 color = tex2D(_MainTex, i.uv);
    119.                 color.rgb = pow(color.rgb, _Gamma);
    120.                 color.rgb = clamp(color.rgb, 0.0, 0.996);
    121.                 return color;
    122.             }
    123.             ENDCG
    124.         }
    125.     }
    126.  
    127.     FallBack Off
    128. }
    129.  
    ARKit Shader

    Code (CSharp):
    1. Shader "inr/AR/ARKit Background Linear"
    2. {
    3.     Properties
    4.     {
    5.         _textureY ("TextureY", 2D) = "white" {}
    6.         _textureCbCr ("TextureCbCr", 2D) = "black" {}
    7.         _Gamma("Gamma", Float) = 2.2
    8.     }
    9.  
    10.     SubShader
    11.     {
    12.         Cull Off
    13.         Tags { "RenderType"="Opaque" }
    14.         LOD 100
    15.  
    16.         Pass
    17.         {
    18.             ZWrite Off
    19.             CGPROGRAM
    20.             #pragma vertex vert
    21.             #pragma fragment frag
    22.  
    23.             #include "UnityCG.cginc"
    24.  
    25.             float4x4 _UnityDisplayTransform;
    26.  
    27.             struct Vertex
    28.             {
    29.                 float4 position : POSITION;
    30.                 float2 texcoord : TEXCOORD0;
    31.             };
    32.  
    33.             struct TexCoordInOut
    34.             {
    35.                 float4 position : SV_POSITION;
    36.                 float2 texcoord : TEXCOORD0;
    37.             };
    38.  
    39.             TexCoordInOut vert (Vertex vertex)
    40.             {
    41.                 TexCoordInOut o;
    42.                 o.position = UnityObjectToClipPos(vertex.position);
    43.  
    44.                 float texX = vertex.texcoord.x;
    45.                 float texY = vertex.texcoord.y;
    46.  
    47.                 o.texcoord.x = (_UnityDisplayTransform[0].x * texX + _UnityDisplayTransform[1].x * (texY) + _UnityDisplayTransform[2].x);
    48.                 o.texcoord.y = (_UnityDisplayTransform[0].y * texX + _UnityDisplayTransform[1].y * (texY) + (_UnityDisplayTransform[2].y));
    49.  
    50.                 return o;
    51.             }
    52.  
    53.             // samplers
    54.             sampler2D _textureY;
    55.             sampler2D _textureCbCr;
    56.             float _Gamma;
    57.  
    58.             fixed4 frag (TexCoordInOut i) : SV_Target
    59.             {
    60.                 // sample the texture
    61.                 float2 texcoord = i.texcoord;
    62.                 float y = tex2D(_textureY, texcoord).r;
    63.                 float4 ycbcr = float4(y, tex2D(_textureCbCr, texcoord).rg, 1.0);
    64.  
    65.                 const float4x4 ycbcrToRGBTransform = float4x4(
    66.                         float4(1.0, +0.0000, +1.4020, -0.7010),
    67.                         float4(1.0, -0.3441, -0.7141, +0.5291),
    68.                         float4(1.0, +1.7720, +0.0000, -0.8860),
    69.                         float4(0.0, +0.0000, +0.0000, +1.0000)
    70.                     );
    71.  
    72.                 float4 color = mul(ycbcrToRGBTransform, ycbcr);
    73.                 color.rgb = pow(color.rgb, float3 (_Gamma, _Gamma, _Gamma));
    74.  
    75.                 return color;
    76.             }
    77.             ENDCG
    78.         }
    79.     }
    80. }
    81.  
    enjoy the beauty of linear color space... ;)
     
  9. SilverBugi

    SilverBugi

    Joined:
    Jan 30, 2017
    Posts:
    4
    First of all, thank you very much for that!
    But when I use your ARCore Shader my Camera has a wrong orientation in Portrait mode.
    In Landscape mode everything works fine but as soon as I flip my phone to Portrait mode the camera still stays in landscape and stretches the picture to portrait.
    Any solution for that?

    Screenshot_20190127-005948.jpg Screenshot_20190127-005958.jpg
     
  10. MarcoElz

    MarcoElz

    Joined:
    Jan 28, 2017
    Posts:
    9
    I am having the same problem that SilverBugi said.
    It only works on "Lanscape Left" Oriantation, other orientations show incorrect camera behaviour.

    But also I have another problem, the detected planes are wrong, when you detect the plane and then move the camera the plane moves like if it has wrong transform values. I mean, like if the plane is far that it should be.

    Any solution?
     
  11. MarcoElz

    MarcoElz

    Joined:
    Jan 28, 2017
    Posts:
    9
    @SilverBugi I found a solution for Android.

    I duplicate the ARCore Shader inside packages\packages.unity.com\com.unity.xr.arcore@1.0.0-preview.23\Runtime\Android\Resources\ARCoreShader.shader

    And just add this line of code at the end of the line 48.
    Source: https://github.com/google-ar/arcore-unity-sdk/issues/268

    //gamma to linear conversion
    gl_FragColor.rgb = pow(gl_FragColor.rgb, vec3(2.2));
     
    SilverBugi likes this.
  12. SilverBugi

    SilverBugi

    Joined:
    Jan 30, 2017
    Posts:
    4
    Thank you very much! It it worked! Didn't had the time to fix it myself, so thanks a lot :)
     
  13. TapUp

    TapUp

    Joined:
    Oct 17, 2016
    Posts:
    2
    ARCore Background Linear - Portrait.


    Code (CSharp):
    1. Shader "ALEXWICK/ARCore/ARBackground(Linear)"
    2. {
    3.     Properties {
    4.         _MainTex ("Texture", 2D) = "white" {}
    5.         _UvTopLeftRight ("UV of top corners", Vector) = (0, 1, 1, 1)
    6.         _UvBottomLeftRight ("UV of bottom corners", Vector) = (0 , 0, 1, 0)
    7.     }
    8.  
    9.     // For GLES3
    10.     SubShader
    11.     {
    12.         Pass
    13.         {
    14.             ZWrite Off
    15.  
    16.             GLSLPROGRAM
    17.  
    18.             #pragma only_renderers gles3
    19.  
    20.             #ifdef SHADER_API_GLES3
    21.             #extension GL_OES_EGL_image_external_essl3 : require
    22.             #endif
    23.  
    24.             uniform vec4 _UvTopLeftRight;
    25.             uniform vec4 _UvBottomLeftRight;
    26.  
    27.             #ifdef VERTEX
    28.  
    29.             varying vec2 textureCoord;
    30.  
    31.             void main()
    32.             {
    33.                 #ifdef SHADER_API_GLES3
    34.                 vec2 uvTop = mix(_UvTopLeftRight.zy, _UvTopLeftRight.xw, gl_MultiTexCoord0.y);
    35.                 vec2 uvBottom = mix(_UvBottomLeftRight.zy, _UvBottomLeftRight.xw, gl_MultiTexCoord0.y);
    36.                 textureCoord = mix(uvTop, uvBottom, gl_MultiTexCoord0.x);
    37.  
    38.                 gl_Position = gl_ModelViewProjectionMatrix * gl_Vertex;
    39.                 #endif
    40.             }
    41.  
    42.             #endif
    43.  
    44.             #ifdef FRAGMENT
    45.             varying vec2 textureCoord;
    46.             uniform samplerExternalOES _MainTex;
    47.  
    48.             void main()
    49.             {
    50.                 #ifdef SHADER_API_GLES3
    51.                 gl_FragColor = texture(_MainTex, textureCoord);
    52.                 //gamma to linear conversion
    53.         gl_FragColor.rgb = pow(gl_FragColor.rgb, vec3(2.2));
    54.                 #endif
    55.             }
    56.  
    57.             #endif
    58.  
    59.             ENDGLSL
    60.         }
    61.     }
    62.  
    63.   Subshader
    64.   {
    65.     Pass
    66.     {
    67.       ZWrite Off
    68.  
    69.       CGPROGRAM
    70.  
    71.       #pragma exclude_renderers gles3
    72.       #pragma vertex vert
    73.       #pragma fragment frag
    74.  
    75.       #include "UnityCG.cginc"
    76.  
    77.       uniform float4 _UvTopLeftRight;
    78.       uniform float4 _UvBottomLeftRight;
    79.  
    80.       struct appdata
    81.       {
    82.         float4 vertex : POSITION;
    83.         float2 uv : TEXCOORD0;
    84.       };
    85.  
    86.       struct v2f
    87.       {
    88.         float2 uv : TEXCOORD0;
    89.         float4 vertex : SV_POSITION;
    90.       };
    91.  
    92.       v2f vert(appdata v)
    93.       {
    94.         float2 uvTop = lerp(_UvTopLeftRight.zy, _UvTopLeftRight.xw, v.uv.y);
    95.         float2 uvBottom = lerp(_UvBottomLeftRight.zy, _UvBottomLeftRight.xw, v.uv.y);
    96.  
    97.         v2f o;
    98.         o.vertex = UnityObjectToClipPos(v.vertex);
    99.         o.uv = lerp(uvTop, uvBottom, v.uv.x);
    100.  
    101.         // Instant preview's texture is transformed differently.
    102.         o.uv = o.uv.yx;
    103.         //o.uv.x = 1.0 - o.uv.x;
    104.         //o.uv.y = 1.0 - o.uv.y;
    105.         return o;
    106.       }
    107.  
    108.       sampler2D _MainTex;
    109.  
    110.       fixed4 frag(v2f i) : SV_Target
    111.       {
    112.            return tex2D(_MainTex, i.uv);
    113.       }
    114.       ENDCG
    115.     }
    116.   }
    117.  
    118.   FallBack Off
    119. }