Stop outlines from merging when overlapping (URP)

Hi, I found this asset called Quick Outline which works and looks great compared to other outline rendering methods I tried with one exception which is that when an object is in front of another the outlines merge together.
My project is filled with objects that use outlines and most of them end out cancelling each other out because of this.



I don't have much experience writing shaders so I was wondering if it's posible to modify the outline shader scripts so it can be rendered per-object in URP and how I could achieve this. Thanks.

These are the 2 shader used for the outline.

OutlineMask.shader:

Code (CSharp):

1. Shader "Custom/Outline Mask" {
2.   Properties {
3.     [Enum(UnityEngine.Rendering.CompareFunction)] _ZTest("ZTest", Float) = 0
4.   }
5.  
6.   SubShader {
7.     Tags {
8.       "Queue" = "Transparent+100"
9.       "RenderType" = "Transparent"
10.     }
11.  
12.     Pass {
13.       Name "Mask"
14.       Cull Off
15.       ZTest [_ZTest]
16.       ZWrite Off
17.       ColorMask 0
18.  
19.       Stencil {
20.         Ref 1
21.         Pass Replace
22.       }
23.     }
24.   }
25. }

OutlineFill.shader:

Code (CSharp):

1. Shader "Custom/Outline Fill" {
2.   Properties {
3.     [Enum(UnityEngine.Rendering.CompareFunction)] _ZTest("ZTest", Float) = 0
4.  
5.     _OutlineColor("Outline Color", Color) = (1, 1, 1, 1)
6.     _OutlineWidth("Outline Width", Range(0, 10)) = 2
7.   }
8.  
9.   SubShader {
10.     Tags {
11.       "Queue" = "Transparent+110"
12.       "RenderType" = "Transparent"
13.       "DisableBatching" = "True"
14.     }
15.  
16.     Pass {
17.       Name "Fill"
18.       Cull Off
19.       ZTest [_ZTest]
20.       ZWrite Off
21.       Blend SrcAlpha OneMinusSrcAlpha
22.       ColorMask RGB
23.  
24.       Stencil {
25.         Ref 1
26.         Comp NotEqual
27.       }
28.  
29.       CGPROGRAM
30.       #include "UnityCG.cginc"
31.  
32.       #pragma vertex vert
33.       #pragma fragment frag
34.  
35.       struct appdata {
36.         float4 vertex : POSITION;
37.         float3 normal : NORMAL;
38.         float3 smoothNormal : TEXCOORD3;
39.         UNITY_VERTEX_INPUT_INSTANCE_ID
40.       };
41.  
42.       struct v2f {
43.         float4 position : SV_POSITION;
44.         fixed4 color : COLOR;
45.         UNITY_VERTEX_OUTPUT_STEREO
46.       };
47.  
48.       uniform fixed4 _OutlineColor;
49.       uniform float _OutlineWidth;
50.  
51.       v2f vert(appdata input) {
52.         v2f output;
53.  
54.         UNITY_SETUP_INSTANCE_ID(input);
55.         UNITY_INITIALIZE_VERTEX_OUTPUT_STEREO(output);
56.  
57.         float3 normal = any(input.smoothNormal) ? input.smoothNormal : input.normal;
58.         float3 viewPosition = UnityObjectToViewPos(input.vertex);
59.         float3 viewNormal = normalize(mul((float3x3)UNITY_MATRIX_IT_MV, normal));
60.  
61.         output.position = UnityViewToClipPos(viewPosition + viewNormal * -viewPosition.z * _OutlineWidth / 1000.0);
62.         output.color = _OutlineColor;
63.  
64.         return output;
65.       }
66.  
67.       fixed4 frag(v2f input) : SV_Target {
68.         return input.color;
69.       }
70.       ENDCG
71.     }
72.   }
73. }

 

The quickest "fix" would be to remove the

Stencil { }

from the fill shader, and change

Cull Off

to

Cull Front

. Though this will make it impossible to use the "visible through walls" option, and show some additional lines on the interior of complex options. Basically it'll work like any traditional toon outline shader using this method.

A more in depth solution would setting the stencil for a single object, rendering the outline for that object, and then clearing the stencil, or assigning unique stencil values for each object.

 

Thanks, that worked!

 

Thanks,how to get shader code。

 

You are using the Quick Outline asset, I guess.
I tried testing your solution but the result was that I had all the shape colored instead of its borders.

Was that what you had ? Did I miss something ?

 

So I did a "sorting layer" system of some sort.

-> Changing "Ref 1" in Stencil to "Ref [_ZRef]", adding "_ZRef("ZRef", Float) = 1" in its properties for both shaders
-> Then editing the Outline.cs script to set this value with the renderQueue

Code (CSharp):

1. public int SortingLayer
2.     {
3.         get { return sortingLayer; }
4.         set
5.         {
6.             sortingLayer = value;
7.             needsUpdate = true;
8.         }
9.     }
10.  
11. // ...
12.  
13.  
14.     [SerializeField]
15.     private int sortingLayer;
16.  
17. // ...
18.  
19. void UpdateMaterialProperties()
20.     {
21.  
22.         // Apply properties according to mode
23.         outlineFillMaterial.SetColor("_OutlineColor", outlineColor);
24.  
25.  
26.         outlineFillMaterial.SetFloat("_ZRef", sortingLayer + 1);
27.         outlineMaskMaterial.SetFloat("_ZRef", sortingLayer + 1);
28.  
29.         outlineMaskMaterial.renderQueue = 3100 + 20 * sortingLayer;
30.         outlineFillMaterial.renderQueue = 3110 + 20 * sortingLayer;
31.  
32. // ....
33.  
34. }

This requires a bit of a setting process but for static elements we want to highlight this does the trick.
I wouldn't recommand this solution for moving objects, though (such as RTS units for example).