Resolved 2D Sprites to not be clipped by 3D Meshes and have Diffused/Lit Sprite shader look?

Using Unity 2020.2.1 and URP 10.2.2 and going for a 2.5D look. (Unity doesn't have built in Lit Sprite shader that work in a 3D environment for URP so you essentially have to choose lighting for 2D or 3D but I need both).

I'm trying to make a shader where Sprites won't be clipped by 3D meshes but still have a Diffused look to them (interact with lights).

Currently I'm trying to use a Diffuse shader, grab its texture with GrabPass and modify it with @bgolus 's VerticalZDepthBillboard shader from here (thank you for this amazing shader btw).

It semi-works but the Sprite loses its pixel perfectness and it grabs parts of the Sprite background as well.



Here's my shader code:

Code (CSharp):

1. Shader "Custom/BillboardVerticalZDepthSpriteDiffuse"
2. {
3.     Properties
4.     {
5.         [PerRendererData] _MainTex ("Sprite Texture", 2D) = "white" {}
6.         _Color ("Tint", Color) = (1,1,1,1)
7.         [MaterialToggle] PixelSnap ("Pixel snap", Float) = 0
8.         [HideInInspector] _RendererColor ("RendererColor", Color) = (1,1,1,1)
9.         [HideInInspector] _Flip ("Flip", Vector) = (1,1,1,1)
10.         [PerRendererData] _AlphaTex ("External Alpha", 2D) = "white" {}
11.         [PerRendererData] _EnableExternalAlpha ("Enable External Alpha", Float) = 0
12.     }
13.     SubShader
14.     {
15.         Tags{
16.             "Queue" = "Transparent"
17.             "IgnoreProjector" = "True"
18.             "RenderType" = "Transparent"
19.             "DisableBatching" = "True"
20.  
21.             /// Diffuse
22.             "PreviewType"="Plane"
23.             "CanUseSpriteAtlas"="True"
24.         }
25.         ZWrite Off
26.         Blend One OneMinusSrcAlpha
27.  
28.         // Diffuse
29.         Cull Off
30.         Lighting Off
31.  
32.         // Diffuse program
33.         CGPROGRAM
34.         #pragma surface surf Lambert vertex:vert nofog nolightmap nodynlightmap keepalpha noinstancing
35.         #pragma multi_compile_local _ PIXELSNAP_ON
36.         #pragma multi_compile _ ETC1_EXTERNAL_ALPHA
37.         #include "UnitySprites.cginc"
38.  
39.         struct Input
40.         {
41.             float2 uv_MainTex;
42.             fixed4 color;
43.         };
44.  
45.         void vert (inout appdata_full v, out Input o)
46.         {
47.             v.vertex = UnityFlipSprite(v.vertex, _Flip);
48.  
49.             #if defined(PIXELSNAP_ON)
50.             v.vertex = UnityPixelSnap (v.vertex);
51.             #endif
52.  
53.             UNITY_INITIALIZE_OUTPUT(Input, o);
54.             o.color = v.color * _Color * _RendererColor;
55.         }
56.  
57.         void surf (Input IN, inout SurfaceOutput o)
58.         {
59.             fixed4 c = SampleSpriteTexture (IN.uv_MainTex) * IN.color;
60.             o.Albedo = c.rgb * c.a;
61.             o.Alpha = c.a;
62.         }
63.         ENDCG
64.  
65.         // Grab Diffused screen data texture
66.         GrabPass
67.         {
68.             "_DiffusedTex"
69.         }
70.  
71.         Pass
72.         {
73.             Blend SrcAlpha OneMinusSrcAlpha
74.          
75.             CGPROGRAM
76.             #pragma vertex vert
77.             #pragma fragment frag
78.             // make fog work
79.             #pragma multi_compile_fog
80.             #include "UnityCG.cginc"
81.             struct appdata
82.             {
83.                 float4 vertex : POSITION;
84.                 float2 grabPos : TEXCOORD0;
85.             };
86.             struct v2f
87.             {
88.                 float4 pos : SV_POSITION;
89.                 float2 grabPos : TEXCOORD0;
90.                 UNITY_FOG_COORDS(1)
91.             };
92.             sampler2D _DiffusedTex;
93.             float4 _MainTex_ST;
94.             float rayPlaneIntersection( float3 rayDir, float3 rayPos, float3 planeNormal, float3 planePos)
95.             {
96.                 float denom = dot(planeNormal, rayDir);
97.                 denom = max(denom, 0.000001); // avoid divide by zero
98.                 float3 diff = planePos - rayPos;
99.                 return dot(diff, planeNormal) / denom;
100.             }
101.             v2f vert(appdata v)
102.             {
103.                 v2f o;
104.                 o.pos = UnityObjectToClipPos(v.vertex);
105.              
106.                 v.grabPos = ComputeGrabScreenPos(o.pos); // get correct texture coordinates
107.                 o.grabPos = v.grabPos.xy;
108.                 // billboard mesh towards camera
109.                 float3 vpos = mul((float3x3)unity_ObjectToWorld, v.vertex.xyz);
110.                 float4 worldCoord = float4(unity_ObjectToWorld._m03, unity_ObjectToWorld._m13, unity_ObjectToWorld._m23, 1);
111.                 float4 viewPos = mul(UNITY_MATRIX_V, worldCoord) + float4(vpos, 0);
112.                 o.pos = mul(UNITY_MATRIX_P, viewPos);
113.                 // calculate distance to vertical billboard plane seen at this vertex's screen position
114.                 float3 planeNormal = normalize(float3(UNITY_MATRIX_V._m20, 0.0, UNITY_MATRIX_V._m22));
115.                 float3 planePoint = unity_ObjectToWorld._m03_m13_m23;
116.                 float3 rayStart = _WorldSpaceCameraPos.xyz;
117.                 float3 rayDir = -normalize(mul(UNITY_MATRIX_I_V, float4(viewPos.xyz, 1.0)).xyz - rayStart); // convert view to world, minus camera pos
118.                 float dist = rayPlaneIntersection(rayDir, rayStart, planeNormal, planePoint);
119.                 // calculate the clip space z for vertical plane
120.                 float4 planeOutPos = mul(UNITY_MATRIX_VP, float4(rayStart + rayDir * dist, 1.0));
121.                 float newPosZ = planeOutPos.z / planeOutPos.w * o.pos.w;
122.                 // use the closest clip space z
123.                 #if defined(UNITY_REVERSED_Z)
124.                 o.pos.z = max(o.pos.z, newPosZ);
125.                 #else
126.                 o.pos.z = min(o.pos.z, newPosZ);
127.                 #endif
128.                 UNITY_TRANSFER_FOG(o,o.vertex);
129.                 return o;
130.             }
131.             fixed4 frag(v2f i) : SV_Target
132.             {
133.                 fixed4 col = tex2D(_DiffusedTex, i.grabPos);
134.                 UNITY_APPLY_FOG(i.fogCoord, col);
135.                 return col;
136.             }
137.             ENDCG
138.         }
139.     }
140.      
141.     Fallback "Transparent/VertexLit"
142. }

 

Ooph. Yeah, no, don't do that. A grab pass "grabs" the entire screen indiscriminately. It is going to get your sprite and everything else in the scene visible where the mesh is. That's the only thing it can do because that's what it's supposed to do. It's possible to abuse for multi-pass materials to get the output of the previous pass, but only really works if all of the passes have exactly the same screen space coverage. And your use case does not.

Also ...

This doesn't actually do proper pixel snapping in a Surface Shader. That's not your fault, it's in a bunch of tutorials and even an official Unity shader. But it totally doesn't work, at all. It changes the position of the vertices a little, but it doesn't do the pixel snap you want. It's confusing because the default sprite shader's code looks very similar, but in a surface shader the

v.vertex

it has access to is the local object space position, but in the default sprite shader

OUT.vertex

is the clip space position, essentially the screen space position. But you can't modify the clip space position directly in a surface shader, so the function can't do anything useful here. And unfortunately someone at Unity made that mistake 5+ years ago and no one noticed it doesn't actually work.

Similarly the vertical z depth billboard shader works by modifying the clip space position, so it also can't be used with surface shaders either. That may be why you tried the above option with the grab pass.

The only way to make the billboard shader work with sprites is to actually update the code to work with sprites. To do that you'll want to look at the default sprite shader's code:
https://github.com/TwoTailsGames/Un.../DefaultResourcesExtra/Sprites-Default.shader
https://github.com/TwoTailsGames/Unity-Built-in-Shaders/blob/master/CGIncludes/UnitySprites.cginc

And if you need lighting, check out the Diffuse lighting with ambient example on this page:
https://docs.unity3d.com/Manual/SL-VertexFragmentShaderExamples.html

 

I see what you mean about not being able to actual modify the pixel perfectness directly in the surface shader.

So I used the resources you shared to write the Billboard Shader to extend off Unity's Sprite Default shader code, and it works well. Note

"DisableBatching"="True"

needs to be set or Sprites on the same sorting layer & same order in layer won't be drawn. Here it is below for anyone who needs it:

Spoiler: BillboardVerticalZDepthSpritesDefault

Code (CSharp):

1. // extending SpritesDefault https://github.com/TwoTailsGames/Unity-Built-in-Shaders/blob/master/DefaultResourcesExtra/Sprites-Default.shader
2. // to include BillboardVerticalZDepth https://forum.unity.com/threads/problem-solving-2d-billboard-sprites-clipping-into-3d-environment.680374/
3.  
4. Shader "Custom/Sprites/BillboardVerticalZDepthSpritesDefault"
5. {
6.     Properties
7.     {
8.         [PerRendererData] _MainTex ("Sprite Texture", 2D) = "white" {}
9.         _Color ("Tint", Color) = (1,1,1,1)
10.         [MaterialToggle] PixelSnap ("Pixel snap", Float) = 0
11.         [HideInInspector] _RendererColor ("RendererColor", Color) = (1,1,1,1)
12.         [HideInInspector] _Flip ("Flip", Vector) = (1,1,1,1)
13.         [PerRendererData] _AlphaTex ("External Alpha", 2D) = "white" {}
14.         [PerRendererData] _EnableExternalAlpha ("Enable External Alpha", Float) = 0
15.     }
16.     SubShader
17.     {
18.         Tags{
19.             "Queue"="Transparent"
20.             "IgnoreProjector"="True"
21.             "RenderType"="Transparent"
22.             "PreviewType"="Plane"
23.             "CanUseSpriteAtlas"="True"
24.  
25.             // must include; otherwise, Sprites on same layer w/ same material will not be drawn
26.             "DisableBatching"="True"
27.         }
28.  
29.         Cull Off
30.         Lighting Off
31.         ZWrite Off
32.         Blend One OneMinusSrcAlpha
33.  
34.         Pass
35.         {
36.         CGPROGRAM
37.             #pragma vertex SpriteBillboardVerticalZDepthVert
38.             #pragma fragment SpriteFrag
39.             #pragma target 2.0
40.             #pragma multi_compile_instancing
41.             #pragma multi_compile_local _ PIXELSNAP_ON
42.             #pragma multi_compile _ ETC1_EXTERNAL_ALPHA
43.             #include "UnitySprites.cginc"
44.  
45.             float RayPlaneIntersection( float3 rayDir, float3 rayPos, float3 planeNormal, float3 planePos)
46.             {
47.                 float denom = dot(planeNormal, rayDir);
48.                 denom = max(denom, 0.000001); // avoid divide by zero
49.                 float3 diff = planePos - rayPos;
50.                 return dot(diff, planeNormal) / denom;
51.             }
52.  
53.             void BillboardVerticalZDepthVert(appdata_t IN, inout v2f OUT)
54.             {
55.                 // billboard mesh towards camera
56.                 float3 vpos = mul((float3x3)unity_ObjectToWorld, IN.vertex.xyz);
57.                 float4 worldCoord = float4(unity_ObjectToWorld._m03, unity_ObjectToWorld._m13, unity_ObjectToWorld._m23, 1);
58.                 float4 viewPos = mul(UNITY_MATRIX_V, worldCoord) + float4(vpos, 0);
59.                 OUT.vertex = mul(UNITY_MATRIX_P, viewPos);
60.  
61.                 // calculate distance to vertical billboard plane seen at this vertex's screen position
62.                 float3 planeNormal = normalize(float3(UNITY_MATRIX_V._m20, 0.0, UNITY_MATRIX_V._m22));
63.                 float3 planePoint = unity_ObjectToWorld._m03_m13_m23;
64.                 float3 rayStart = _WorldSpaceCameraPos.xyz;
65.                 float3 rayDir = -normalize(mul(UNITY_MATRIX_I_V, float4(viewPos.xyz, 1.0)).xyz - rayStart); // convert view to world, minus camera pos
66.                 float dist = RayPlaneIntersection(rayDir, rayStart, planeNormal, planePoint);
67.  
68.                 // calculate the clip space z for vertical plane
69.                 float4 planeOutPos = mul(UNITY_MATRIX_VP, float4(rayStart + rayDir * dist, 1.0));
70.                 float newPosZ = planeOutPos.z / planeOutPos.w * OUT.vertex.w;
71.              
72.                 // use the closest clip space z
73.                 #if defined(UNITY_REVERSED_Z)
74.                 OUT.vertex.z = max(OUT.vertex.z, newPosZ);
75.                 #else
76.                 OUT.vertex.z = min(OUT.vertex.z, newPosZ);
77.                 #endif
78.             }
79.  
80.             v2f SpriteBillboardVerticalZDepthVert(appdata_t IN)
81.             {
82.                 v2f OUT;
83.  
84.                 UNITY_SETUP_INSTANCE_ID (IN);
85.                 UNITY_INITIALIZE_VERTEX_OUTPUT_STEREO(OUT);
86.              
87.                 OUT.vertex = UnityFlipSprite(IN.vertex, _Flip);
88.                 OUT.vertex = UnityObjectToClipPos(OUT.vertex);
89.                 OUT.texcoord = IN.texcoord;
90.                 OUT.color = IN.color * _Color * _RendererColor;
91.  
92.                 BillboardVerticalZDepthVert(IN, OUT);
93.              
94.                 #ifdef PIXELSNAP_ON
95.                 OUT.vertex = UnityPixelSnap (OUT.vertex);
96.                 #endif
97.  
98.                 return OUT;
99.             }
100.         ENDCG
101.         }
102.     }
103. }
104.  

As for adding lighting. I took a stab at it but can't seem to wrap my head around how to get the vertex function they provided to work for Sprites? It seems the main issue is coming from me not being able to calculate a logical

worldNormal

.

I'm considering switching to URP so I could potentially do this through the ShaderGraph there.

Code (CSharp):

1. v2f vert (appdata_base v)
2. {
3.     v2f o;
4.     o.vertex = UnityObjectToClipPos(v.vertex);
5.     o.uv = v.texcoord;
6.     half3 worldNormal = UnityObjectToWorldNormal(v.normal);
7.     half nl = max(0, dot(worldNormal, _WorldSpaceLightPos0.xyz));
8.     o.diff = nl * _LightColor0;
9.  
10.     // the only difference from previous shader:
11.     // in addition to the diffuse lighting from the main light,
12.     // add illumination from ambient or light probes
13.     // ShadeSH9 function from UnityCG.cginc evaluates it,
14.     // using world space normal
15.     o.diff.rgb += ShadeSH9(half4(worldNormal,1));
16.     return o;
17. }

Anyways, thanks for all the resources and the reply, it was really helpful!!

 

Shader Graph has the same problem as Surface Shaders in that you can't modify the clip space position directly.

 

I'm a complete newbie when it comes to graphics but just to make sure I'm understanding this correctly -- with the Vertical Z Depth code you shared, you are making a vertical plane and then casting a ray from the sprite to that plane to make a vertical "implied position." And then you replace the clip position with this "implied position" so the correct pixels will be clipped?

And Shader Graph just doesn't have the capability to modify clip space positions? (This seems to be the case from my few hours of experimenting.)

Alright I'll have to just continue looking into writing a shader.

 

I'm casting a ray from the vertex of the mesh the sprite is rendered on to find the clip space depth of a (mostly) vertical plane that is at the sprite's origin. The clip space position is a special 4 dimensional screen space used for interpolating values between vertices in a way that keeps them linear. But it can be abused, which is what I'm doing here. You can change just the z value in clip space to adjust the screen space depth without affecting how the x and y values are interpolated. The benefit of this is it keeps the UVs from being distorted. As far as the GPU is concerned, at least for the UVs, it's still a flat plane.

In Surface Shaders and Shader Graph you can modify the 3D vertex positions and put the vertices in the same position on screen and in depth, but because we'd be actually warping the 3D position and not the 4D position the texture UVs will distort in weird ways. Kind of like this:

 

Thank you so much for this explanation, it clears things up and really pointed me in the right direction on which rabbit holes to avoid.

I saw Unity's URP shaders actually use vertex/fragment shaders vs. surface shaders to handle their lighting, so I referenced NotSlot's and tategarringer's solution from this thread, and used your Vertical Z Depth function in Unity's Vertex shader in LitForwardPass.hlsl.

The results look pretty good!! (left: after, right: before)



I made a repo with the code https://github.com/strawberryjamnbutter/unity-URP-2.5D-lit-shader; hopefully this can help someone. I used the 2019.3release branch just to prototype but will try updating it sooner or later.

I've tested performance and it doesn't seem to affect fps much at all.

Do you have any insights on major pitfalls with this solution? And this will be fine with both Mac and Windows/OpenGL and DirectX? I did see you mention before it'll be impossible to calculate the (mostly) vertical plane from a topdown view.

Really appreciate all the help on this along with your other posts; seriously this is a pretty cool and neat solution! I'll mark this thread as resolved for now.

 

One gotcha to be mindful of is shadow receiving / casting. Though it appears to be working fine in the above example, directional shadows on opaque objects probably work a bit better in this use case due to them using the depth texture, which presumably is also being offset. Other shadow types, and directional shadows on mobile, use the world position passed to the fragment shader pass. Since that's not being offset, you might see your sprites receive shadows like their heads are still embedded in walls (because they are).

 

Ah got it, that's really good to know. I'm just targeting desktop and for now just using directional lights for shadows so hopefully won't run into this. *fingers crossed*

 

This is amazing! It's exactly what I need for my sprites, and the shading is a huge plus. Unfortunately, none of the other texture maps in the shader seem to work. I'm completely new to the shader scene, so I may just be missing something, but if the Normal and Emission maps worked this would be absolute perfection for my current project. Any insights would be appreciated. And thanks again for putting this together.

 

I haven't actually tested for those use cases, but the shader is based on Unity's 3D Lit Shader and the main change was replacing

_BaseMap

with

_MainTex

(since it's required for Sprites). The main problem here was needing a way to incorporate bgolus' VerticalZDepthBillboard shader to rewrite the clip space depth, which was possible with their Lit Shader since it used plain vertex & fragment shaders. Hopefully this is helpful in some way!

 

I managed to get things working, but I was wondering if this sort of functionality would work as a customs function node in shader graph. I would love to be able to use that tool, but before I even attempt to convert the code into a custom node I wanted to see if it was even possible. I know that bgolus already said this:

But I wasn't sure if that was absolute, or just a limitation with the standard shader graph nodes.

 

Could you please post on here what you did to make it work? Or make a pull request on the repo? And I'll add it to the repo once I have some spare time, so other people can make use of it in the future?

The Lit behavior will work in Shader graph but the adjustment of the sprite's clip position to avoid clipping will not, since the clip space adjustment needs to be made in the vertex shader, which is exposed in URP's shaders but not shader graph (when I checked). That's one main reason why I based this shader off of URP's.

 

I essentially just did what you did, but I used the Simple Lit shader as the foundation and altered SimpleLitForwardPass.hlsl to include the stuff for billboarding and clipping.

So, again, I'm new to shaders... Will creating a custom node with the clipping code not work at all then. I mean, my project is using URP and the clipping adjustment works in code. Isn't the idea of being able to make custom nodes intended to add functionality like this to shader graph? Am I missing something with how shader graph handles stuff?

 

When you're editing the "__ForwardPass.hlsl" files you have direct access to the

.positionCS

value, which is the clip space position the vertex shader outputs. Surface Shaders and Shader Graph explicitly do not let you touch those values from within their sandbox. As I said above, you can only modify the vertex's 3D position, but then that is converted to clip space in a part of the generated shader that you cannot modify from a Surface Shader's code or a Shader Graph's nodes. You can of course modify the resulting generated shader manually, or in the case of Shader Graph modify the shader code generation systems or project's .hlsl files that Shader Graph is calling, but then you're not doing it in the Shader Graph / Surface Shader.

 

So basically, unless Unity exposes that stuff to Shader Graph, it's not currently possible in any way to do this sort of thing inside the Shader Graph interface, even with a custom node?

That being the case, it sounds like the closest thing would be making a shader in Shader Graph, then generating its code and adding the clipping stuff to that.

Is that the gist of it?

 

I'm no expert in Shaders either, I think bgolus is though. But right, when I was checking for a solution using Shader Graph, the interface will only allow you to change their vertex position as an output as seen below.

And I chose to base it off URP's Lit Shader because they don't use Surface shaders, so I could edit the plain vertex shader in LitForwardPass and adjust the clip space in there with bgolus' clipping solution.



And to your last question, I think that sounds good if your workflow requires Shader Graph.

 

Correct.

Yep.


The one caveat being that the HDRP has an optional Depth Offset input for the Shader Graph Master Stack, which while not as efficient as directly modifying the vertex output, would at least let you achieve the same visual result. This does not exist for the URP (yet?).

 

Thanks for the clarification. Hopefully, we'll see this and more stuff like it become available in Shader Graph soon.

That option in HDRP sounds tempting though. How much less efficient are we talking?

 

Very interesting read, thank you gentlemen !

I know this is quite rude but @bgolus seems to be the only expert on the subject around, so I'll give it a try xD

I've been trying for days to escape the 2D Renderer pipeline because I need to be able to create a "blur plane" that would blur any sprites behind it. It seems impossible in this renderer, should I switch to URP ?
But in URP I can't seem to be able to find a simple sprite-lit shader...

I'm actually ready to pay for a setup that would work ^^
Yep, I'm kinda desperate !