Linear Color Space for Camera Background

I just started using the foundation framework to see how it works out.

During testing I wondered how to use a proper linear camera background shader instead of the default one. Seems like the background image is to bright when setting the player settings to linear.

I tried to plug the arkit camera shader into a material and use this as override, but this gives me black background.

Is there a sample shader available that will work?

 

Is there any solution for this problem? I'm having it too!

 

I'm using AR Foundation version 1.0.0-preview.17 and ARKit XR Plugin version 1.0.0-preview.13. Here's the solution I'm using.

Create a new shader by coping the shader code from

Packages/ARKit XR Plugin/Runtime/iOS/Resource/ARKitShader

.

Change this line (near the end of the file):

Code (CSharp):

1. return mul(ycbcrToRGBTransform, ycbcr);

...to this...

Code (CSharp):

1. return pow(mul(ycbcrToRGBTransform, ycbcr), 2.2);

Then create a new material that uses that shader and assign that material to your AR Camera's "AR Camera Background" component's "Custom Material" property.

 

@Krxtopher Thanks for that hint. But how would I do this if I have to support both Android and iOS?

(I am looking for a way to create my own custom shader to dim the background video programmatically. If there's any other way to do this, I would be happy to know about it.)

 

You can programmatically set the custom material based on platform conditional defines.

 

Hey guys, I'm having troubles with that.
I have tried with this solution (https://github.com/Unity-Technologies/arfoundation-samples/issues/46)
and this other (https://github.com/google-ar/arcore-unity-sdk/issues/268)
but stills, everytime I launch the android app I get a black screen from the begining, it doesnt even load the Unity splash screen. :/

I have checked on LogCat and I'm getting this error:

[EGL] Failed to create window surface: EGL_BAD_MATCH: Arguments are inconsitent (for example, a valid context requires buffers not supplied by a valid surface)

Do you guys have any idea of what might be it or what I might be doing wrong?

Thanks

 

Just in case anyone else is facing this problem, we fixed it enabling on "ProjectSettings/Resolution and Presentation" the checkbox "Use 32-bit Display Buffer" (Unity 2018.2.14)

 

Hi all,

to support linear color space for android and ios you need a script to do the job and two materials with a dedicated shader for ARCore and ARKit (as @tdmowrer suggested).

you can use this script. attach it to the game object where your ARCameraBackground script resides in your AR scene.

Code (CSharp):

1. //
2. //
3. // purpose: set camera background material depending on platform used
4. //
5. //
6. // author: ge
7. //
8. // (c) 2019 innovation.rocks consulting gmbh
9. //
10.  
11. using UnityEngine;
12. using UnityEngine.XR.ARFoundation;
13.  
14. [RequireComponent(typeof(ARCameraBackground))]
15. public class ARCameraPlatformBackground : MonoBehaviour
16. {
17.     public Material _ARCoreBackground;
18.     public Material _ARKitBackground;
19.  
20.     void Start ()
21.     {
22.         ARCameraBackground arcbg = GetComponent<ARCameraBackground> ();
23.         arcbg.useCustomMaterial = true;
24. #if UNITY_IOS
25.         arcbg.customMaterial = _ARKitBackground;
26. #else
27.         arcbg.customMaterial = _ARCoreBackground;
28. #endif
29.     }
30. }
31.  

make two materials for ARCore and ARKit with the following shaders attached


ARCore Shader

Code (CSharp):

1. Shader "inr/AR/ARCore Background Linear"
2. {
3.     Properties {
4.         _MainTex ("Texture", 2D) = "white" {}
5.         _UvTopLeftRight ("UV of top corners", Vector) = (0, 1, 1, 1)
6.         _UvBottomLeftRight ("UV of bottom corners", Vector) = (0 , 0, 1, 0)
7.         _Gamma("Gamma", Float) = 2.2
8.     }
9.  
10.     // For GLES3
11.     SubShader
12.     {
13.         Pass
14.         {
15.             ZWrite Off
16.  
17.             GLSLPROGRAM
18.  
19.             #pragma only_renderers gles3
20.  
21.             #ifdef SHADER_API_GLES3
22.             #extension GL_OES_EGL_image_external_essl3 : require
23.             #endif
24.  
25.             uniform vec4 _UvTopLeftRight;
26.             uniform vec4 _UvBottomLeftRight;
27.  
28.             #ifdef VERTEX
29.  
30.             varying vec2 textureCoord;
31.  
32.             void main()
33.             {
34.                 #ifdef SHADER_API_GLES3
35.                 vec2 uvTop = mix(_UvTopLeftRight.xy, _UvTopLeftRight.zw, gl_MultiTexCoord0.x);
36.                 vec2 uvBottom = mix(_UvBottomLeftRight.xy, _UvBottomLeftRight.zw, gl_MultiTexCoord0.x);
37.                 textureCoord = mix(uvTop, uvBottom, gl_MultiTexCoord0.y);
38.  
39.                 gl_Position = gl_ModelViewProjectionMatrix * gl_Vertex;
40.                 #endif
41.             }
42.  
43.             #endif
44.  
45.             #ifdef FRAGMENT
46.             varying vec2 textureCoord;
47.             uniform samplerExternalOES _MainTex;
48.             uniform float _Gamma;
49.  
50.             void main()
51.             {
52.                 #ifdef SHADER_API_GLES3
53.                 vec4 color = texture(_MainTex, textureCoord);
54.  
55.                 color.rgb = pow(color.rgb, vec3(_Gamma, _Gamma, _Gamma));
56.                 color.rgb = clamp(color.rgb, 0.0, 0.996);
57.  
58.                 gl_FragColor = color;
59.                 #endif
60.             }
61.  
62.             #endif
63.  
64.             ENDGLSL
65.         }
66.     }
67.  
68.     Subshader
69.     {
70.         Pass
71.         {
72.             ZWrite Off
73.  
74.             CGPROGRAM
75.  
76.             #pragma exclude_renderers gles3
77.             #pragma vertex vert
78.             #pragma fragment frag
79.  
80.             #include "UnityCG.cginc"
81.  
82.             uniform float4 _UvTopLeftRight;
83.             uniform float4 _UvBottomLeftRight;
84.             uniform float _Gamma;
85.  
86.             struct appdata
87.             {
88.                 float4 vertex : POSITION;
89.                 float2 uv : TEXCOORD0;
90.             };
91.  
92.             struct v2f
93.             {
94.                 float2 uv : TEXCOORD0;
95.                 float4 vertex : SV_POSITION;
96.             };
97.  
98.             v2f vert(appdata v)
99.             {
100.                 float2 uvTop = lerp(_UvTopLeftRight.xy, _UvTopLeftRight.zw, v.uv.x);
101.                 float2 uvBottom = lerp(_UvBottomLeftRight.xy, _UvBottomLeftRight.zw, v.uv.x);
102.  
103.                 v2f o;
104.                 o.vertex = UnityObjectToClipPos(v.vertex);
105.                 o.uv = lerp(uvTop, uvBottom, v.uv.y);
106.  
107.                 // Instant preview's texture is transformed differently.
108.                 o.uv = o.uv.yx;
109.                 o.uv.x = 1.0 - o.uv.x;
110.  
111.                 return o;
112.             }
113.  
114.             sampler2D _MainTex;
115.  
116.             fixed4 frag(v2f i) : SV_Target
117.             {
118.                 fixed4 color = tex2D(_MainTex, i.uv);
119.                 color.rgb = pow(color.rgb, _Gamma);
120.                 color.rgb = clamp(color.rgb, 0.0, 0.996);
121.                 return color;
122.             }
123.             ENDCG
124.         }
125.     }
126.  
127.     FallBack Off
128. }
129.  

ARKit Shader

Code (CSharp):

1. Shader "inr/AR/ARKit Background Linear"
2. {
3.     Properties
4.     {
5.         _textureY ("TextureY", 2D) = "white" {}
6.         _textureCbCr ("TextureCbCr", 2D) = "black" {}
7.         _Gamma("Gamma", Float) = 2.2
8.     }
9.  
10.     SubShader
11.     {
12.         Cull Off
13.         Tags { "RenderType"="Opaque" }
14.         LOD 100
15.  
16.         Pass
17.         {
18.             ZWrite Off
19.             CGPROGRAM
20.             #pragma vertex vert
21.             #pragma fragment frag
22.  
23.             #include "UnityCG.cginc"
24.  
25.             float4x4 _UnityDisplayTransform;
26.  
27.             struct Vertex
28.             {
29.                 float4 position : POSITION;
30.                 float2 texcoord : TEXCOORD0;
31.             };
32.  
33.             struct TexCoordInOut
34.             {
35.                 float4 position : SV_POSITION;
36.                 float2 texcoord : TEXCOORD0;
37.             };
38.  
39.             TexCoordInOut vert (Vertex vertex)
40.             {
41.                 TexCoordInOut o;
42.                 o.position = UnityObjectToClipPos(vertex.position);
43.  
44.                 float texX = vertex.texcoord.x;
45.                 float texY = vertex.texcoord.y;
46.  
47.                 o.texcoord.x = (_UnityDisplayTransform[0].x * texX + _UnityDisplayTransform[1].x * (texY) + _UnityDisplayTransform[2].x);
48.                 o.texcoord.y = (_UnityDisplayTransform[0].y * texX + _UnityDisplayTransform[1].y * (texY) + (_UnityDisplayTransform[2].y));
49.  
50.                 return o;
51.             }
52.  
53.             // samplers
54.             sampler2D _textureY;
55.             sampler2D _textureCbCr;
56.             float _Gamma;
57.  
58.             fixed4 frag (TexCoordInOut i) : SV_Target
59.             {
60.                 // sample the texture
61.                 float2 texcoord = i.texcoord;
62.                 float y = tex2D(_textureY, texcoord).r;
63.                 float4 ycbcr = float4(y, tex2D(_textureCbCr, texcoord).rg, 1.0);
64.  
65.                 const float4x4 ycbcrToRGBTransform = float4x4(
66.                         float4(1.0, +0.0000, +1.4020, -0.7010),
67.                         float4(1.0, -0.3441, -0.7141, +0.5291),
68.                         float4(1.0, +1.7720, +0.0000, -0.8860),
69.                         float4(0.0, +0.0000, +0.0000, +1.0000)
70.                     );
71.  
72.                 float4 color = mul(ycbcrToRGBTransform, ycbcr);
73.                 color.rgb = pow(color.rgb, float3 (_Gamma, _Gamma, _Gamma));
74.  
75.                 return color;
76.             }
77.             ENDCG
78.         }
79.     }
80. }
81.  

enjoy the beauty of linear color space...

 

First of all, thank you very much for that!
But when I use your ARCore Shader my Camera has a wrong orientation in Portrait mode.
In Landscape mode everything works fine but as soon as I flip my phone to Portrait mode the camera still stays in landscape and stretches the picture to portrait.
Any solution for that?

 

I am having the same problem that SilverBugi said.
It only works on "Lanscape Left" Oriantation, other orientations show incorrect camera behaviour.

But also I have another problem, the detected planes are wrong, when you detect the plane and then move the camera the plane moves like if it has wrong transform values. I mean, like if the plane is far that it should be.

Any solution?

 

@SilverBugi I found a solution for Android.

I duplicate the ARCore Shader inside packages\packages.unity.com\com.unity.xr.arcore@1.0.0-preview.23\Runtime\Android\Resources\ARCoreShader.shader

And just add this line of code at the end of the line 48.
Source: https://github.com/google-ar/arcore-unity-sdk/issues/268

//gamma to linear conversion

gl_FragColor.rgb = pow(gl_FragColor.rgb, vec3(2.2));

 

Thank you very much! It it worked! Didn't had the time to fix it myself, so thanks a lot

 

ARCore Background Linear - Portrait.

Code (CSharp):

1. Shader "ALEXWICK/ARCore/ARBackground(Linear)"
2. {
3.     Properties {
4.         _MainTex ("Texture", 2D) = "white" {}
5.         _UvTopLeftRight ("UV of top corners", Vector) = (0, 1, 1, 1)
6.         _UvBottomLeftRight ("UV of bottom corners", Vector) = (0 , 0, 1, 0)
7.     }
8.  
9.     // For GLES3
10.     SubShader
11.     {
12.         Pass
13.         {
14.             ZWrite Off
15.  
16.             GLSLPROGRAM
17.  
18.             #pragma only_renderers gles3
19.  
20.             #ifdef SHADER_API_GLES3
21.             #extension GL_OES_EGL_image_external_essl3 : require
22.             #endif
23.  
24.             uniform vec4 _UvTopLeftRight;
25.             uniform vec4 _UvBottomLeftRight;
26.  
27.             #ifdef VERTEX
28.  
29.             varying vec2 textureCoord;
30.  
31.             void main()
32.             {
33.                 #ifdef SHADER_API_GLES3
34.                 vec2 uvTop = mix(_UvTopLeftRight.zy, _UvTopLeftRight.xw, gl_MultiTexCoord0.y);
35.                 vec2 uvBottom = mix(_UvBottomLeftRight.zy, _UvBottomLeftRight.xw, gl_MultiTexCoord0.y);
36.                 textureCoord = mix(uvTop, uvBottom, gl_MultiTexCoord0.x);
37.  
38.                 gl_Position = gl_ModelViewProjectionMatrix * gl_Vertex;
39.                 #endif
40.             }
41.  
42.             #endif
43.  
44.             #ifdef FRAGMENT
45.             varying vec2 textureCoord;
46.             uniform samplerExternalOES _MainTex;
47.  
48.             void main()
49.             {
50.                 #ifdef SHADER_API_GLES3
51.                 gl_FragColor = texture(_MainTex, textureCoord);
52.                 //gamma to linear conversion
53.         gl_FragColor.rgb = pow(gl_FragColor.rgb, vec3(2.2));
54.                 #endif
55.             }
56.  
57.             #endif
58.  
59.             ENDGLSL
60.         }
61.     }
62.  
63.   Subshader
64.   {
65.     Pass
66.     {
67.       ZWrite Off
68.  
69.       CGPROGRAM
70.  
71.       #pragma exclude_renderers gles3
72.       #pragma vertex vert
73.       #pragma fragment frag
74.  
75.       #include "UnityCG.cginc"
76.  
77.       uniform float4 _UvTopLeftRight;
78.       uniform float4 _UvBottomLeftRight;
79.  
80.       struct appdata
81.       {
82.         float4 vertex : POSITION;
83.         float2 uv : TEXCOORD0;
84.       };
85.  
86.       struct v2f
87.       {
88.         float2 uv : TEXCOORD0;
89.         float4 vertex : SV_POSITION;
90.       };
91.  
92.       v2f vert(appdata v)
93.       {
94.         float2 uvTop = lerp(_UvTopLeftRight.zy, _UvTopLeftRight.xw, v.uv.y);
95.         float2 uvBottom = lerp(_UvBottomLeftRight.zy, _UvBottomLeftRight.xw, v.uv.y);
96.  
97.         v2f o;
98.         o.vertex = UnityObjectToClipPos(v.vertex);
99.         o.uv = lerp(uvTop, uvBottom, v.uv.x);
100.  
101.         // Instant preview's texture is transformed differently.
102.         o.uv = o.uv.yx;
103.         //o.uv.x = 1.0 - o.uv.x;
104.         //o.uv.y = 1.0 - o.uv.y;
105.         return o;
106.       }
107.  
108.       sampler2D _MainTex;
109.  
110.       fixed4 frag(v2f i) : SV_Target
111.       {
112.            return tex2D(_MainTex, i.uv);
113.       }
114.       ENDCG
115.     }
116.   }
117.  
118.   FallBack Off
119. }

 

ARFoundation 3.0.0-preview.3 now supports both linear and gamma color spaces.