Custom Lighting Function breaks dithering?

I'm trying to implement into my shader for a character, that if the camera gets too close to the player, the chara starts to dither away based on camera distance -> there are many example projects like this: https://www.ronja-tutorials.com/2019/05/11/dithering.html

My problem occurs once i add a custom lighting function (that i need for my shader / using deferred rendering). Pixel still get clipped like normal, but the shadows of the object stay in the air and block everything else. Which breaks this transparency effect-



Also it seems like only the color value gets clipped when using custom lighting. The shadows stay there, as well as the reflection-

Has somebody an idea on how to fix this?

Shader without custom lighting (from Alex Ocias):

Code (CSharp):

1.     Shader "Ocias/Diffuse (Stipple Transparency)" {
2.     Properties {
3.         _MainTex ("Base (RGB)", 2D) = "white" {}
4.         _Transparency ("Transparency", Range(0,1)) = 1.0
5.     }
6.     SubShader {
7.         Tags { "RenderType"="Opaque" }
8.         LOD 150
9.     CGPROGRAM
10.     #pragma surface surf Lambert noforwardadd
11.     sampler2D _MainTex;
12.     struct Input {
13.         float2 uv_MainTex;
14.         float4 screenPos;
15.     };
16.     half _Transparency;
17.     void surf (Input IN, inout SurfaceOutput o) {
18.         fixed4 c = tex2D(_MainTex, IN.uv_MainTex);
19.         o.Albedo = c.rgb;
20.         o.Alpha = c.a;
21.         // Screen-door transparency: Discard pixel if below threshold.
22.         float4x4 thresholdMatrix =
23.         {  1.0 / 17.0,  9.0 / 17.0,  3.0 / 17.0, 11.0 / 17.0,
24.           13.0 / 17.0,  5.0 / 17.0, 15.0 / 17.0,  7.0 / 17.0,
25.            4.0 / 17.0, 12.0 / 17.0,  2.0 / 17.0, 10.0 / 17.0,
26.           16.0 / 17.0,  8.0 / 17.0, 14.0 / 17.0,  6.0 / 17.0
27.         };
28.         float4x4 _RowAccess = { 1,0,0,0, 0,1,0,0, 0,0,1,0, 0,0,0,1 };
29.         float2 pos = IN.screenPos.xy / IN.screenPos.w;
30.         pos *= _ScreenParams.xy; // pixel position
31.         clip(_Transparency - thresholdMatrix[fmod(pos.x, 4)] * _RowAccess[fmod(pos.y, 4)]);
32.     }
33.     ENDCG
34.     }
35.     Fallback "Mobile/VertexLit"
36.     }

Same shader just with custom lighting:

Code (CSharp):

1.         Shader "Ocias/Diffuse (Stipple Transparency)" {
2.         Properties {
3.             _MainTex ("Base (RGB)", 2D) = "white" {}
4.             _Transparency ("Transparency", Range(0,1)) = 1.0
5.         }
6.         SubShader {
7.             Tags { "RenderType"="Opaque" }
8.             LOD 150
9.         CGPROGRAM
10.         #pragma surface surf Stylized
11.         sampler2D _MainTex;
12.    
13.         float4 LightingStylized(SurfaceOutput s, float3 lightDir, half3 viewDir, float shadowAttenuation){
14.            
15.                 float towardsLight = dot(s.Normal, lightDir);
16.                 float4 color = towardsLight * shadowAttenuation;
17.                 color.a = s.Alpha;
18.                 return color;
19.             }
20.    
21.         struct Input {
22.             float2 uv_MainTex;
23.             float4 screenPos;
24.         };
25.         half _Transparency;
26.         void surf (Input IN, inout SurfaceOutput o) {
27.             fixed4 c = tex2D(_MainTex, IN.uv_MainTex);
28.             o.Albedo = c.rgb;
29.             o.Alpha = c.a;
30.             // Screen-door transparency: Discard pixel if below threshold.
31.             float4x4 thresholdMatrix =
32.             {  1.0 / 17.0,  9.0 / 17.0,  3.0 / 17.0, 11.0 / 17.0,
33.               13.0 / 17.0,  5.0 / 17.0, 15.0 / 17.0,  7.0 / 17.0,
34.                4.0 / 17.0, 12.0 / 17.0,  2.0 / 17.0, 10.0 / 17.0,
35.               16.0 / 17.0,  8.0 / 17.0, 14.0 / 17.0,  6.0 / 17.0
36.             };
37.             float4x4 _RowAccess = { 1,0,0,0, 0,1,0,0, 0,0,1,0, 0,0,0,1 };
38.             float2 pos = IN.screenPos.xy / IN.screenPos.w;
39.             pos *= _ScreenParams.xy; // pixel position
40.             clip(_Transparency - thresholdMatrix[fmod(pos.x, 4)] * _RowAccess[fmod(pos.y, 4)]);
41.         }
42.         ENDCG
43.         }
44.         Fallback "Mobile/VertexLit"
45.         }
46.  

 

I believe the issue is that no custom shadow pass is being generated with your current settings. On your #pragma line, add

addshadow

and see if that fixes things for you?

 

Surface shaders using the Lambert shading model still work with the deferred rendering path. It's identical to using a Standard Specular shader with the smoothness set to 0.0 and the specular color set to black.

Surface shaders using a custom shading model are not compatible with deferred and will be rendered as part of the forward shading pass.

However the way deferred and forward shading interact, specifically when it comes to the main directional light's shadows, is a little funny.


For traditional forward rendering, Unity renders out a camera depth texture and the directional shadow maps, both of which use the same shadow caster pass from the shaders. Unity then casts the shadow maps onto the camera depth texture by deriving the per pixel world position from the depth texture. There's no official way to differentiate in the shader between if it's the depth texture or the shadow map that is currently being rendered. The only option I've found (without additional scripting) is to test if the current projection matrix is orthographic or perspective projection.
https://forum.unity.com/threads/dif...dow-caster-and-shadow-receiver-in-5-2.362653/

With scripting you can use a command buffer to set a global variable or keyword during the depth texture pass, but it's not really necessary unless your main camera is using an orthographic projection.


How does this relate to deferred, and the interaction between deferred and forward rendering? The deferred rendering path uses the shader's deferred pass output to get the depth texture used for rendering the shadows (and all deferred lighting), so any clip() or discard calls you use the surf() function get properly handled. However opaque forward rendered objects are injected into the gbuffers using the same shadow caster pass used to generate the depth texture! So the existing issue of the shadow caster pass getting used for both shadow casting and shadow receiving continues to exist when using the deferred rendering path for any opaque objects that don't use deferred rendering.

The solution is, as @Invertex mentioned, to include addshadow on the surface shader's #pragma surf line. That alone will mean the shadow map will also be dithered, which you may or may not want. If you don't want the shadow map to be dithered, you'll have to use the perspective / orthographic test in the above link in the surf() function and skip the dithering if currently rendering to the shadow map (ie: currently using orthographic projection).

 

@Invertex @bgolus Thanks you two for the answers and holy cow, bgolus, that explanation was incredible detailed! Thank you so much! This really cleared up a lot for me regarding how forward and deferred work togetherxD (Sadly writing deferred shaders isn't well documented anywhere, alteast i can't remember seeing it. I remember a thread were you talked about it a bit tho, but it was nothing official regarding deferred shaders, i thinkxD)

I also just gone ahead a tried the addshadow pragma and unexpected stuff happenedxD So everything actually stays the same regarding the shadows until i turn the transperency to 0. The shadow then just vanishes completlyxD

On the other hand i noticed, that i could also switch the objects render queue to transparent, which gets rid of shadows completly on the object itself, not the shadow it throws. (which is what i want to achieve ideally) It also removes shadows on the object from other objects too which is noticeable, but acceptable i guess. But I'm not sure about performance regarding this hack. I'd have to use a script for changing the queue everytime the player gets near the character.

This effect of blending out characters is seen often in games, like in Mario Odyssey, they use the dither technique on Mario and other objects(but with everything working, maybe they use pure deferred shaders?) In Splatoon 2 or Zelda BotW, they use full on smooth transparency, which is interesting on several levels, because (1) they have to use the extra pass technique to counteract this weird transparency issue, where you can see the object itself in glitchy ways, because of depthsorting and (2) the characters still keep the shadows of other objects on them. I'm not sure if they all found smart solutions for shadows on transparency or if this is just a unity problem, but i don't think so-

 

Oh yeah. I seem to remember there's a bug with Surface shaders that the screenPos value isn't actually filled for the shadow caster passes. That's a real pain. Basically you have to supply the screen position manually via a custom vertex function, or you could write your own custom shadow caster pass instead of using addshadow.

 

Mario Odyssey uses dithering for some transparency (even different dither patterns for different situations), real transparency for other things. Mainly anything that's lit and shadowed is always opaque and dithered. They also used a "shadows cast on depth" setup, but even more interesting the depth texture they cast shadows on doesn't necessarily match the resolution of the actual game view, and the resolution of both are adjusted dynamically as needed for performance.

Both of these games appear to use more straightforward methods for shadow receiving. Zelda specifically is most likely still some form of deferred renderer, but they have fully transparent objects able to sample the shadow maps directly. That is actually possible with Unity with some custom shaders and hacks, and isn't supported really only because Unity chose to disable support for it (for legacy performance reasons) rather than any explicit technical limitation.

 

Atleast calculating the screenpos manually didn't change anything- And i think i implemented the shadowcaster idea wrong, but i'm also not sure, on how to clip the shadow itself, if its basically just a function in the fragment shader(?)

Code (CSharp):

1.  Shader "Ocias/Diffuse (Stipple Transparency)" {
2.         Properties {
3.             _MainTex ("Base (RGB)", 2D) = "white" {}
4.             _Transparency ("Transparency", Range(0,1)) = 1.0
5.         }
6.         SubShader {
7.             Tags { "RenderType"="Opaque" }
8.             LOD 150
9.  
10.         CGPROGRAM
11.         #pragma  surface surf Stylized vertex:vert
12.         #pragma target 3.5
13.  
14.         sampler2D _MainTex;
15.  
16.         float4 LightingStylized(SurfaceOutput s, float3 lightDir, half3 viewDir, float shadowAttenuation){
17.          
18.                 float towardsLight = dot(s.Normal, lightDir);
19.                 float4 color = towardsLight * shadowAttenuation;
20.                 color.a = s.Alpha;
21.                 return color;
22.                
23.             }
24.  
25.         struct Input {
26.             float2 uv_MainTex;
27.             float4 screenPos;
28.         };
29.  
30.         void vert (inout appdata_full v, out Input o) {
31.             UNITY_INITIALIZE_OUTPUT(Input,o);
32.             o.screenPos = ComputeScreenPos(UnityObjectToClipPos(v.vertex));
33.         }
34.  
35.         half _Transparency;
36.         void surf (Input IN, inout SurfaceOutput o) {
37.             fixed4 c = tex2D(_MainTex, IN.uv_MainTex);
38.             o.Albedo = c.rgb;
39.             o.Alpha = c.a;
40.             // Screen-door transparency: Discard pixel if below threshold.
41.             float4x4 thresholdMatrix =
42.             {  1.0 / 17.0,  9.0 / 17.0,  3.0 / 17.0, 11.0 / 17.0,
43.               13.0 / 17.0,  5.0 / 17.0, 15.0 / 17.0,  7.0 / 17.0,
44.                4.0 / 17.0, 12.0 / 17.0,  2.0 / 17.0, 10.0 / 17.0,
45.               16.0 / 17.0,  8.0 / 17.0, 14.0 / 17.0,  6.0 / 17.0
46.             };
47.             float4x4 _RowAccess = { 1,0,0,0, 0,1,0,0, 0,0,1,0, 0,0,0,1 };
48.             float2 pos = IN.screenPos.xy / IN.screenPos.w;
49.             pos *= _ScreenParams.xy; // pixel position
50.             clip(_Transparency - thresholdMatrix[fmod(pos.x, 4)] * _RowAccess[fmod(pos.y, 4)]);
51.         }
52.         ENDCG
53.        
54.         // shadow caster rendering pass, implemented manually
55.         // using macros from UnityCG.cginc
56.         Pass
57.         {
58.             Tags {"LightMode"="ShadowCaster"}
59.  
60.             CGPROGRAM
61.             #pragma vertex vert
62.             #pragma fragment frag
63.             #pragma multi_compile_shadowcaster
64.             #include "UnityCG.cginc"
65.  
66.             struct v2f {
67.                 V2F_SHADOW_CASTER;
68.                 float4 screenPos : TEXCOORD0;
69.             };
70.  
71.             v2f vert(appdata_base v)
72.             {
73.                 v2f o;
74.                 TRANSFER_SHADOW_CASTER_NORMALOFFSET(o)
75.                 o.screenPos = ComputeScreenPos(UnityObjectToClipPos(v.vertex));
76.                 return o;
77.             }
78.  
79.             float4 frag(v2f i) : SV_Target
80.             {
81.                 SHADOW_CASTER_FRAGMENT(i)
82.  
83.             }
84.             ENDCG
85.         }
86.         }
87.         //Fallback "Mobile/VertexLit"
88.         }
89.  

 

Yeah, i actually noticed that too! Like on some hard edges you can sometimes see the lit color of the meshes, because the shadow res doesnt match the aliased edge pattern of the objectsxD

Oh ok! Gotta look out for those. I have a water shader, that really could use some shadows on top to block the sun light reflections in shadows.