Question Outline shader does not render to depth

I have an outline shader that renders a separate pass pushed back in z-space (and offset in screenpos) that get PPv2 (post processing stack v2) effects drawing on top of it (DOF and SSAO).



After some investigating in the forums I'm pretty sure the issue is that the outline shader does not render to depth and the solution is to render a shadow casting pass (see https://forum.unity.com/threads/cus...epth-and-normals-for-post-processing.1156454/ for example).

I've tried implementing a shadow pass with the same displacement code as in the outline pass but as I understand it, the shadow pass needs mesh displacement set in v.vertex (see here https://forum.unity.com/threads/ver...rt/frag shader with shadows on offset vertice).

This code renders to depth but not displaced. If set v.vertex before SHADOW_CASTER_FRAGMENT to some random value it does indeed displace but I'm not sure how to translate the outline displacement to v.vertex.

Thanks!

(Original shader (that I have made changes to) by the amazing https://twitter.com/minionsart)

Code (CSharp):

1. Shader "Outline"  {
2.     Properties {
3.         _Color ("Main Color", Color) = (.5,.5,.5,1)
4.         _MainTex ("Base (RGB)", 2D) = "white" { }
5.     }
6.    
7.     CGINCLUDE
8.     #include "UnityCG.cginc"
9.    
10.    
11.     struct appdata {
12.         float4 vertex : POSITION;
13.         float3 normal : NORMAL;
14.         float4 texcoord : TEXCOORD0; // texture coordinates
15.     };
16.  
17.     struct v2f {
18.         float4 pos : SV_POSITION;
19.         UNITY_FOG_COORDS(0)
20.         fixed4 color : COLOR;
21.         float4 screenPos : TEXCOORD0;//
22.     };
23.    
24.     uniform float _Outline, _XOffset, _YOffset, _OutlineZ, _Brightness;
25.     sampler2D _MainTex;
26.    
27.     v2f vert(appdata v) {
28.         v2f o;
29.         o.pos = UnityObjectToClipPos(v.vertex);
30.         o.screenPos = ComputeScreenPos(o.pos);//
31.  
32.         _Brightness =  2.4;
33.         _Outline =  0.00129;
34.         _OutlineZ = 0.005;
35.         _YOffset = -0.445;
36.         _XOffset = 0;
37.  
38.         float3 norm = normalize(mul ((float3x3)UNITY_MATRIX_IT_MV, v.vertex));
39.         float2 offset = TransformViewToProjection(norm.xy) + (float2(_XOffset, _YOffset) * 20);
40.         float4 tex = tex2Dlod(_MainTex, float4(v.texcoord.xy, 0, 0));
41.        
42.         #ifdef UNITY_Z_0_FAR_FROM_CLIPSPACE //to handle recent standard asset package on older version of unity (before 5.5)
43.             o.pos.xy += offset * UNITY_Z_0_FAR_FROM_CLIPSPACE(o.pos.z) * _Outline;
44.         #else
45.             o.pos.xy += offset * o.pos.z * _Outline;
46.         #endif
47.         o.pos.z -= (o.screenPos.z * _OutlineZ);
48.  
49.             o.color = tex * _Brightness * 1.1 * (unity_AmbientGround + unity_AmbientSky + unity_AmbientEquator);
50.  
51.         UNITY_TRANSFER_FOG(o, o.pos);
52.         return o;
53.     }
54.     ENDCG
55.     SubShader {
56.         Tags {
57.             "RenderType" = "Opaque"
58.             "Queue" = "Geometry"
59.         }
60.         UsePass "Base/FORWARD"
61.         Pass {
62.             Tags { "LightMode"="ShadowCaster" }
63.            
64.             CGPROGRAM
65.             #pragma vertex vert
66.             #pragma fragment frag
67.             #pragma multi_compile_shadowcaster
68.             #include "UnityCG.cginc"
69.            
70.             struct v2f2 {
71.                 V2F_SHADOW_CASTER;
72.                 float4 screenPos : TEXCOORD0;//
73.             };
74.            
75.             v2f2 vert(appdata_base v)
76.             {
77.                 v2f2 o;
78.                 o.pos = UnityObjectToClipPos(v.vertex);
79.                 o.screenPos = ComputeScreenPos(o.pos);//
80.  
81.                 _Outline =  0.00129;
82.                 _OutlineZ = 0.005;
83.                 _YOffset = -0.445;
84.                 _XOffset = 0;
85.  
86.                 float3 norm = normalize(mul ((float3x3)UNITY_MATRIX_IT_MV, v.vertex));
87.                 float2 offset = TransformViewToProjection(norm.xy) + (float2(_XOffset, _YOffset) * 20);
88.                 float4 tex = tex2Dlod(_MainTex, float4(v.texcoord.xy, 0, 0));
89.                
90.                 #ifdef UNITY_Z_0_FAR_FROM_CLIPSPACE // to handle recent standard asset package on older version of unity (before 5.5)
91.                     o.pos.xy += offset * UNITY_Z_0_FAR_FROM_CLIPSPACE(o.pos.z) * _Outline;
92.                 #else
93.                     o.pos.xy += offset * o.pos.z * _Outline;
94.                 #endif
95.  
96.                 TRANSFER_SHADOW_CASTER_NORMALOFFSET(o)
97.                 return o;
98.             }
99.            
100.             float4 frag(v2f i) : SV_Target
101.             {
102.                 SHADOW_CASTER_FRAGMENT(i)
103.             }
104.             ENDCG
105.         }
106.  
107.         Pass {
108.             Tags {
109.                 "LightMode" = "Always"
110.             }
111.             Name "OUTLINE"
112.             ZWrite On
113.             ColorMask RGB
114.             Blend One Zero
115.  
116.             CGPROGRAM
117.             #pragma vertex vert
118.             #pragma fragment frag
119.             #pragma multi_compile_fog
120.             fixed4 frag(v2f i) : SV_Target
121.             {
122.                 UNITY_APPLY_FOG(i.fogCoord, i.color);
123.                 return i.color;
124.             }
125.             ENDCG
126.         }
127.  
128.     }
129.  
130.     Fallback "Diffuse"
131. }

 

Yep, you're correct that the problem is the lack of a shadow caster pass. However the solution for this isn't going to be all that straightforward. The problem is outline shaders like this are multi-pass shaders; one pass for the outline and one pass for the direct lit object. Unity's shadow system is setup to only render the first shadow caster pass it finds in a shader, it will not render any additional shadow caster passes as usually these are superfluous. But the only way to get the outline to render to the depth is to have it have it's own shadow caster pass. Additionally you usually don't want the outline to cast a shadow itself, and there's no (easy) way to limit the shadow caster pass to only render during the camera depth pass.

The easiest solution to this is use separate mesh renderers for the outline and the object rather than having a single mesh & shader that does both. Have an outline only shader & material you assign to one mesh, and use a regular shader & material on the other. On the outline only mesh disable shadow casting on the mesh renderer component itself.

However be warned this won't entirely fix the SSAO problem depending on which SSAO you're using. Some SSAO methods only use the camera depth texture, at which point this should all work (though be warned the outline may still receiving AO since it's being drawn during the opaque passes). Some SSAO methods also use the camera depth normals texture, which gets rendered in a completely different way. That uses the built in Internal-DepthNormalsTexture.shader which has several different passes in it with unique "RenderType" tags.
https://github.com/TwoTailsGames/Un...rcesExtra/Internal-DepthNormalsTexture.shader

You'd want to give your outline only shader it's own unique "RenderType", and then add a pass using that tag and doing the outline offset to a copy of that Internal-DepthNormalsTexture.shader. You can then tell Unity to use that updated shader by setting an override in the project's Graphics settings.