Resolved Wrong shadow map sampling depending on camera pose

Hi,

I wrote a custom vertex / fragment shader for the built-in render pipeline which includes shadow receiving. The shader generally works, but it turned out that with some camera poses, the shadow receiving (implemented with UNITY_SHADOW_ATTENUATION) breaks. When this happens, instead of proper shadow receiving, it looks like the cascade shadow map atlas becomes visible in the mesh instead. This happens on Linux with OpenGL, with Unity 2021.3.15f1. Here are some images:

Correct shadow receiving (although the quality isn't great, but that is another issue):


Wrong shadow receiving after moving the camera, with what looks like the parts of the shadow map atlas highlighted:


So far, I had no success trying to fix this, so I wanted to ask whether anyone has seen this issue before and knows how to fix it.

For reference, the custom shader is embedded below. In principle, it follows the shadow receiving instructions from here: https://docs.unity3d.com/Manual/SL-VertexFragmentShaderExamples.html
I upgraded to the macros with the UNITY_ prefix according to that forum post, but that did not fix the issue: https://forum.unity.com/threads/sha...latform-vert-frag-shader.701438/#post-6246163

Shader pass definition:

Code (CSharp):

1. Shader "Shadername"
2. {
3.     Properties
4.     {
5.         _Ambient ("Ambient", Float) = 0.2
6.     }
7.    
8.     SubShader
9.     {
10.         Tags { "RenderType" = "Opaque" }
11.         LOD 100
12.        
13.         // The vertex shaders mirror the local x coordinates of the mesh to adapt the meshes,
14.         // which use a right-handed coordinate system, to Unity's left-handed coordinate system.
15.         // Because of that, the side for back-face culling must be flipped as well.
16.         Cull Front
17.        
18.         // (other passes ...)
19.        
20.         Pass
21.         {
22.             CGPROGRAM
23.             #pragma vertex vert
24.             #pragma fragment frag
25.            
26.             #pragma multi_compile_instancing
27.             #pragma multi_compile_fog
28.            
29.             // Shadow receiving
30.             #pragma multi_compile_fwdbase nolightmap nodirlightmap nodynlightmap novertexlight
31.            
32.             #include "Builtin_forward.hlsl"
33.             ENDCG
34.         }
35.        
36.         // (other passes ...)
37.     }
38. }
39.  

Content of Builtin_forward.hlsl:

Spoiler: Shader code (long)

Code (CSharp):

1. #include "UnityCG.cginc"
2. #include "AutoLight.cginc"
3.  
4. struct appdata {
5.   // Note: Automatic conversion of position and texcoord to float at this point works with Metal, but fails with OpenGL.
6.   //       Thus, these attributes must be left as uintX here even though they get directly converted to floatX.
7.   uint4 position    : POSITION;
8.   uint2 texcoord    : TEXCOORD0;
9.   uint4 nodeIndices : TEXCOORD1;
10.   uint4 nodeWeights : TEXCOORD2;
11.   UNITY_VERTEX_INPUT_INSTANCE_ID
12. };
13.  
14. struct shadowinput {
15.   float4 vertex;
16. };
17.  
18. struct v2f {
19.   float4 pos               : SV_POSITION;  // must be named "pos" for TRANSFER_SHADOW()
20.   centroid float2 texcoord : TEXCOORD0;
21.   float3 worldPos          : TEXCOORD1;
22.   UNITY_FOG_COORDS(2)        // put fog coordinates into TEXCOORD2 if enabled
23.   UNITY_SHADOW_COORDS(3)     // put shadows data into TEXCOORD3 if enabled;
24.                              // NOTE: As of the time of writing, Unity's documentation does not seem to mention UNITY_SHADOW_COORDS at all,
25.                              //       but according to https://catlikecoding.com/unity/tutorials/rendering/part-17/, it is a newer version of SHADOW_COORDS().
26.   UNITY_VERTEX_OUTPUT_STEREO
27. };
28.  
29. #define K 4
30.  
31. #if defined(SHADER_TARGET_GLSL) || defined(SHADER_API_GLCORE) || defined(SHADER_API_GLES) || defined(SHADER_API_GLES3)
32.   Texture2D_float<uint4> deformationState;  // Note: The _float suffix makes this a `highp` sampler in OpenGL ES, which is required for it to work on Android
33.   int deformationStateWidth;  // Note: We pass this in addition to `deformationState` itself since querying the size from the texture object did not work when compiled to GLSL
34. #else
35.   StructuredBuffer<float> deformationState;
36. #endif
37.  
38. texture2D textureLuma;
39. texture2D textureChromaU;
40. texture2D textureChromaV;
41.  
42. float4 bboxMin;
43. float4 vertexFactor;
44. float4 textureLumaSize;  // Note: Do not call this "textureSize", as this may be reserved in some shader languages
45.  
46. float _Ambient;
47.  
48. inline float3 DecodePosition(uint4 position) {
49.   return bboxMin.xyz + vertexFactor.xyz * float3(position.xyz);
50. }
51.  
52. v2f vert(appdata vi) {
53.   v2f o;
54.   UNITY_SETUP_INSTANCE_ID(vi);
55.   UNITY_INITIALIZE_OUTPUT(v2f, o);
56.   UNITY_INITIALIZE_VERTEX_OUTPUT_STEREO(o);
57.  
58.   o.texcoord = float2(0.5 / 65536.0, 0.5 / 65536.0) + float2(1.0 / 65536.0, 1.0 / 65536.0) * float2(vi.texcoord);
59.  
60.   float weightsAsFloat[K];
61.  
62.   // De-quantize the weights to float
63.   for (int k = 0; k < K; ++ k) {
64.     uint nodeWeightK = vi.nodeWeights[k];
65.     weightsAsFloat[k] =
66.         (nodeWeightK == 1) ? (0.5 * (0.5 / 254.)) :
67.         ((nodeWeightK == 255) ? (253.75 / 254.) :
68.           ((max(nodeWeightK, 1) - 1) / 254.));
69.   }
70.  
71.   // Re-normalize the weights
72.   float weightFactor = 1. / (weightsAsFloat[0] + weightsAsFloat[1] + weightsAsFloat[2] + weightsAsFloat[3]);  // assumes K == 4
73.  
74.   for (int j = 0; j < K; ++ j) {
75.     weightsAsFloat[j] *= weightFactor;
76.   }
77.  
78.   // Compute the deformed vertex position
79.   float3 originalPosition = DecodePosition(vi.position);
80.   float3 deformedPosition = float3(0, 0, 0);
81.  
82.   for (int n = 0; n < K; ++ n) {
83.     #if defined(SHADER_TARGET_GLSL) || defined(SHADER_API_GLCORE) || defined(SHADER_API_GLES) || defined(SHADER_API_GLES3)
84.       uint nodeIndex = vi.nodeIndices[n];
85.       uint deformationStateWidthUnsigned = deformationStateWidth;
86.      
87.       int texelIdx = 3 * nodeIndex;
88.       int texelFetchY = texelIdx / deformationStateWidthUnsigned;
89.       float4 deformationStateA = asfloat(deformationState.Load(int3(texelIdx - texelFetchY * deformationStateWidthUnsigned, texelFetchY, /*lod*/ 0)));
90.      
91.       ++ texelIdx;
92.       texelFetchY = texelIdx / deformationStateWidthUnsigned;
93.       float4 deformationStateB = asfloat(deformationState.Load(int3(texelIdx - texelFetchY * deformationStateWidthUnsigned, texelFetchY, /*lod*/ 0)));
94.      
95.       ++ texelIdx;
96.       texelFetchY = texelIdx / deformationStateWidthUnsigned;
97.       float4 deformationStateC = asfloat(deformationState.Load(int3(texelIdx - texelFetchY * deformationStateWidthUnsigned, texelFetchY, /*lod*/ 0)));
98.      
99.       deformedPosition +=
100.           weightsAsFloat[n] *
101.           (float3(deformationStateA[0] * originalPosition.x + deformationStateA[3] * originalPosition.y + deformationStateB[2] * originalPosition.z,
102.                   deformationStateA[1] * originalPosition.x + deformationStateB[0] * originalPosition.y + deformationStateB[3] * originalPosition.z,
103.                   deformationStateA[2] * originalPosition.x + deformationStateB[1] * originalPosition.y + deformationStateC[0] * originalPosition.z) +
104.           float3(deformationStateC[1], deformationStateC[2], deformationStateC[3]));
105.     #else
106.       uint baseIdx = 12 * vi.nodeIndices[n];
107.      
108.       deformedPosition +=
109.           weightsAsFloat[n] *
110.           (float3(deformationState[baseIdx + 0] * originalPosition.x + deformationState[baseIdx + 3] * originalPosition.y + deformationState[baseIdx + 6] * originalPosition.z,
111.                   deformationState[baseIdx + 1] * originalPosition.x + deformationState[baseIdx + 4] * originalPosition.y + deformationState[baseIdx + 7] * originalPosition.z,
112.                   deformationState[baseIdx + 2] * originalPosition.x + deformationState[baseIdx + 5] * originalPosition.y + deformationState[baseIdx + 8] * originalPosition.z) +
113.           float3(deformationState[baseIdx + 9], deformationState[baseIdx + 10], deformationState[baseIdx + 11]));
114.     #endif
115.   }
116.  
117.   deformedPosition.x = -1 * deformedPosition.x;
118.  
119.   o.worldPos = mul(UNITY_MATRIX_M, deformedPosition);
120.   o.pos = UnityObjectToClipPos(deformedPosition);
121.  
122.   // Compute fog amount from clip space position
123.   UNITY_TRANSFER_FOG(o, o.pos);
124.  
125.   // Note: For some platforms (e.g., Android), the TRANSFER_SHADOW() macro assumes that the vertex position can be accessed as "v.vertex".
126.   //       Since our input attribute is however encoded as uint4 (which is not suitable for direct use by the macro),
127.   //       we use a custom helper struct "shadowinput" here to provide the input for the macro instead.
128.   // Note: As of the time of writing, Unity still does not seem to have any documentation on UNITY_TRANSFER_SHADOW(),
129.   //       despite it apparently behaving better than the old TRANSFER_SHADOW().
130.   //       There is this forum post of a Unity employee:
131.   //       https://forum.unity.com/threads/shadow-problem-of-android-platform-vert-frag-shader.701438/#post-6246163
132.   shadowinput v;
133.   v.vertex = float4(deformedPosition, 1.0);
134.   UNITY_TRANSFER_SHADOW(o, vi.texcoord);  // vi.texcoord is used as dummy since we don't need lightmap coords
135.  
136.   return o;
137. }
138.  
139. half3 SampleRGB(uint2 xy) {
140.   half luma = textureLuma.Load(int3(xy, 0)).x;
141.   half chromaU = textureChromaU.Load(int3(xy / 2, 0)).x;
142.   half chromaV = textureChromaV.Load(int3(xy / 2, 0)).x;
143.  
144.   luma -= 16. / 255.;
145.   chromaU -= 128. / 255.;
146.   chromaV -= 128. / 255.;
147.  
148.   const half3 srgb = clamp(half3(
149.       1.164f * luma                    + 1.596f * chromaV,
150.       1.164f * luma - 0.392f * chromaU - 0.813f * chromaV,
151.       1.164f * luma + 2.017f * chromaU                   ), 0.f, 1.f);
152.  
153.   #if UNITY_COLORSPACE_GAMMA
154.     return srgb;
155.   #else
156.     return GammaToLinearSpace(srgb);
157.   #endif
158. }
159.  
160. fixed4 frag(v2f i) : SV_Target {
161.   // Simple bilinear interpolation (in linear RGB space, after converting the four input colors from YUV to linear RGB).
162.   UNITY_SETUP_STEREO_EYE_INDEX_POST_VERTEX(i);
163.  
164.   float2 xy = textureLumaSize.xy * i.texcoord - float2(0.5, 0.5);
165.   uint2 baseXY = uint2(xy);
166.   float2 fract = frac(xy);
167.  
168.   const half3 topLeft = SampleRGB(baseXY);
169.   const half3 topRight = SampleRGB(uint2(baseXY.x + 1, baseXY.y));
170.   const half3 bottomLeft = SampleRGB(uint2(baseXY.x, baseXY.y + 1));
171.   const half3 bottomRight = SampleRGB(uint2(baseXY.x + 1, baseXY.y + 1));
172.  
173.   // Bilinear interpolation.
174.   const half topLeftWeight     = (1.0f - fract.x) * (1.0f - fract.y);
175.   const half topRightWeight    =         fract.x  * (1.0f - fract.y);
176.   const half bottomLeftWeight  = (1.0f - fract.x) *         fract.y;
177.   const half bottomRightWeight =         fract.x  *         fract.y;
178.   fixed4 col = fixed4(
179.       topLeftWeight * topLeft +
180.       topRightWeight * topRight +
181.       bottomLeftWeight * bottomLeft +
182.       bottomRightWeight * bottomRight,
183.       1.0f);
184.  
185.   // Compute shadow attenuation (1.0 = fully lit, 0.0 = fully shadowed).
186.   // Regarding UNITY_SHADOW_ATTENUATION(), see the comment on UNITY_TRANSFER_SHADOW() above.
187.   col.rgb *= _Ambient + (1.0 - _Ambient) * UNITY_SHADOW_ATTENUATION(i, i.worldPos);
188.  
189.   // Apply fog
190.   UNITY_APPLY_FOG(i.fogCoord, col);
191.  
192.   return col;
193. }
194.  
195.  

The relevant parts of this should be the following ones:

Code (CSharp):

1. struct shadowinput {
2.   float4 vertex;
3. };
4.  
5. struct v2f {
6.   float4 pos               : SV_POSITION;  // must be named "pos" for TRANSFER_SHADOW()
7.   centroid float2 texcoord : TEXCOORD0;
8.   float3 worldPos          : TEXCOORD1;
9.   UNITY_FOG_COORDS(2)        // put fog coordinates into TEXCOORD2 if enabled
10.   UNITY_SHADOW_COORDS(3)     // put shadows data into TEXCOORD3 if enabled;
11.                              // NOTE: As of the time of writing, Unity's documentation does not seem to mention UNITY_SHADOW_COORDS at all,
12.                              //       but according to https://catlikecoding.com/unity/tutorials/rendering/part-17/, it is a newer version of SHADOW_COORDS().
13.   UNITY_VERTEX_OUTPUT_STEREO
14. };

Code (CSharp):

1.   // Note: For some platforms (e.g., Android), the TRANSFER_SHADOW() macro assumes that the vertex position can be accessed as "v.vertex".
2.   //       Since our input attribute is however encoded as uint4 (which is not suitable for direct use by the macro),
3.   //       we use a custom helper struct "shadowinput" here to provide the input for the macro instead.
4.   // Note: As of the time of writing, Unity still does not seem to have any documentation on UNITY_TRANSFER_SHADOW(),
5.   //       despite it apparently behaving better than the old TRANSFER_SHADOW().
6.   //       There is this forum post of a Unity employee:
7.   //       https://forum.unity.com/threads/shadow-problem-of-android-platform-vert-frag-shader.701438/#post-6246163
8.   shadowinput v;
9.   v.vertex = float4(deformedPosition, 1.0);
10.   UNITY_TRANSFER_SHADOW(o, vi.texcoord);  // vi.texcoord is used as dummy since we don't need lightmap coords

Code (CSharp):

1.   // Compute shadow attenuation (1.0 = fully lit, 0.0 = fully shadowed).
2.   // Regarding UNITY_SHADOW_ATTENUATION(), see the comment on UNITY_TRANSFER_SHADOW() above.
3.   col.rgb *= _Ambient + (1.0 - _Ambient) * UNITY_SHADOW_ATTENUATION(i, i.worldPos);

To stress this again, the shader works fine most of the time, the shadow receiving just sometimes breaks depending on the camera pose. The mesh itself always looks fine.

Thanks in advance for any hints!

 

I think that I found the problem: The shader pass was missing the "LightMode" = "ForwardBase" tag. After adding that, shadow receiving seems to work consistently.

Code (csharp):

1.         Pass
2.         {
3.             Tags { "LightMode" = "ForwardBase" }
4.          
5.             // ...
6.         }