Stencil Effect Shader

I am trying to get an effect and I am not sure what technique I should be going with. I have tried depth mask and stencil effect and can't seem to fine tune it to get the desired effect.

I have a single mesh that I would like to move around on a table and would like the mesh to only show up on the table. I am using the stencil effect below. However it seems to be extending past the mask shader.


I found the following shader scripts on a previous forum that seemed to be the closest to the effect that I would like to achieve. If you have any suggestions that would be great.

Code (CSharp):

1. Shader "Custom/Mask" {
2.     SubShader{
3.         Tags{ "RenderType" = "Transparent" }
4.         Stencil{
5.         Ref 1
6.         Comp always
7.         Pass replace
8.     }
9.  
10.         CGPROGRAM
11. #pragma surface surf Lambert alpha
12.  
13.         struct Input {
14.         fixed3 Albedo;
15.     };
16.  
17.     void surf(Input IN, inout SurfaceOutput o) {
18.         o.Albedo = fixed3(1, 1, 1);
19.         o.Alpha = 0;
20.     }
21.     ENDCG
22.     }
23.         FallBack "Diffuse"
24. }

Code (CSharp):

1. Shader "Custom/StencilEffect" {
2.     Properties{
3.         _MainTex("Base (RGB)", 2D) = "white" {}
4.     }
5.         SubShader{
6.         Tags{ "RenderType" = "Opaque" }
7.  
8.         Stencil{
9.         Ref 1
10.         Comp equal
11.         Pass Keep
12.     }
13.  
14.         CGPROGRAM
15. #pragma surface surf Lambert
16.  
17.         sampler2D _MainTex;
18.  
19.     struct Input {
20.         float2 uv_MainTex;
21.     };
22.  
23.     void surf(Input IN, inout SurfaceOutput o) {
24.         half4 c = tex2D(_MainTex, IN.uv_MainTex);
25.         o.Albedo = c.rgb;
26.         o.Alpha = c.a;
27.     }
28.     ENDCG
29.     }
30.         FallBack "Diffuse"
31. }

 

-UPDATE-
Changed the technique. I am still learning on how to achieve this look. I found this article upon searching for an answer.
https://alastaira.wordpress.com/2013/09/07/super-mario-galaxy-reveal-hidden-platforms-cg-shader/

It has helped me achieve this look.

However I am struggling to retain the shader properties of the previous look.
Does anyone have any suggestions?

Shader:

Code (CSharp):

1. // Upgrade NOTE: replaced '_Object2World' with 'unity_ObjectToWorld'
2. // Upgrade NOTE: replaced 'mul(UNITY_MATRIX_MVP,*)' with 'UnityObjectToClipPos(*)'
3.  
4. Shader "Custom/Proximity"
5. {
6.     Properties{
7.         _MainTex("Base (RGB)", 2D) = "white" {} // Regular object texture
8.         _PlayerPosition("Player Position", vector) = (0,0,0,0) // The location of the player - will be set by script
9.         _VisibleDistance("Visibility Distance", float) = 10.0 // How close does the player have to be to make object visible
10.         _OutlineWidth("Outline Width", float) = 3.0 // Used to add an outline around visible area a la Mario Galaxy
11.         _OutlineColour("Outline Colour", color) = (1.0,1.0,0.0,1.0) // Colour of the outline
12.         _Glossiness("Smoothness", Range(0,1)) = 0.5
13.         _Metallic("Metallic", Range(0,1)) = 0.0
14.     }
15.         SubShader{
16.         Tags{ "RenderType" = "Transparent" "Queue" = "Transparent" }
17.         Pass{
18.         Blend SrcAlpha OneMinusSrcAlpha
19.  
20.         CGPROGRAM
21.  
22. #pragma vertex vert
23.  
24. #pragma fragment frag
25.  
26.     // Access the shaderlab properties
27.     //uniform sampler2D _MainTex;
28.     sampler2D _MainTex;
29.     uniform float4 _PlayerPosition;
30.     uniform float _VisibleDistance;
31.     uniform float _OutlineWidth;
32.     uniform fixed4 _OutlineColour;
33.  
34.     struct Input
35.     {
36.         float2 uv_MainTex;
37.     };
38.  
39.     half _Glossiness;
40.     half _Metallic;
41.     fixed4 _Color;
42.  
43.     // Input to vertex shader
44.     struct vertexInput {
45.  
46.         float4 vertex : POSITION;
47.         float4 texcoord : TEXCOORD0;
48.     };
49.     // Input to fragment shader
50.     struct vertexOutput {
51.         float4 pos : SV_POSITION;
52.         float4 position_in_world_space : TEXCOORD0;
53.         float4 tex : TEXCOORD1;
54.     };
55.  
56.     // Add instancing support for this shader. You need to check 'Enable Instancing' on materials that use the shader.
57.     // See https://docs.unity3d.com/Manual/GPUInstancing.html for more information about instancing.
58.     // #pragma instancing_options assumeuniformscaling
59.     //UNITY_INSTANCING_BUFFER_START(Props)
60.     // put more per-instance properties here
61.     //UNITY_INSTANCING_BUFFER_END(Props)
62.  
63.     // VERTEX SHADER
64.     vertexOutput vert(vertexInput input)
65.     {
66.         vertexOutput output;
67.         output.pos = UnityObjectToClipPos(input.vertex);
68.         output.position_in_world_space = mul(unity_ObjectToWorld, input.vertex);
69.         output.tex = input.texcoord;
70.         return output;
71.     }
72.  
73.     // FRAGMENT SHADER
74.     float4 frag(vertexOutput input) : COLOR
75.     {
76.         // Calculate distance to player position
77.         float dist = distance(input.position_in_world_space, _PlayerPosition);
78.  
79.     // Return appropriate colour
80.     if (dist < _VisibleDistance)
81.         {
82.         return tex2D(_MainTex, float4(input.tex)); // Visible
83.         }
84.     else if (dist < _VisibleDistance + _OutlineWidth)
85.         {
86.         return _OutlineColour; // Edge of visible range
87.         }
88.     else
89.         {
90.         discard;
91.         float4 tex = tex2D(_MainTex, float4(input.tex)); // Outside visible range
92.         tex.a = 0;
93.         return tex;
94.         }
95.     }
96.  
97.  
98.         ENDCG
99.     } // End Pass
100.     } // End Subshader
101.         FallBack "Diffuse"
102. } // End Shader
103.  

Code (CSharp):

1. using System.Collections;
2. using System.Collections.Generic;
3. using UnityEngine;
4.  
5. public class ProximityXray : MonoBehaviour
6. {
7.     public Transform player;
8.     Renderer render;
9.     void Start()
10.     {
11.         render = gameObject.GetComponent<Renderer>();
12.     }
13.     void Update()
14.     {
15.         render.sharedMaterial.SetVector("_PlayerPosition", player.position);
16.     }
17. }
18.  

 

The thing to understand about stencils is they're a screen space thing. The stencil's mask are the pixels that object appears on screen, not in 3d space. Hence the object showing up "behind" where you expected in your first attempt.

The second example is testing the world space position of each pixel as it's being rendered to determine its distance from a point, and hiding those pixels when they're too far away. It's also using a very simple vertex fragment shader to do this, with no lighting or shading.

The first example is using a surface shader, which is a vertex fragment shader generator. You get to write a small portion of the actual shader within some restrictions, and Unity puts together the rest of the shader to add lighting based on the parameters you set. Everything that the second example is doing can be easily done in a surface shader. It even already has a built in worldPos Input variable you can use to get the per pixel world position.

I would recommend reading through this tutorial if you haven't already:
https://www.alanzucconi.com/2015/06/10/a-gentle-introduction-to-shaders-in-unity3d/


Also, your first example shader is actually doing the distance calculation incorrectly. It's comparing the distance between two float4 values. Generally you're not going to be caring about four dimensional space distances. It should be a distance between only the xyz components of the world position and player position. In a surface shader, worldPos is already only ever a float3, but you should likely also define your _PlayerPosition as a float3 instead of a float4 in the shader, or use _PlayerPosition.xyz. You should also be using alpha testing instead of alpha blending for this kind of effect.

 

Perfect! Thanks so much! Yeah I need some help understanding the difference in shaders. I will study up on the link.

 

Thanks again for your help. I found some more help on another forum from you and just wanted to post my findings just in case someone else goes down this rabbit hole. I stumbled across this guys tutorials after reading the tutorial in the link you gave me.

http://halisavakis.com/my-take-on-shaders-spherical-mask-dissolve/

Here is where I ended up with the shader. You suggested I add an addshadow to the #pragma surface line which fixed the clipped surfaces causing shadows.

Code (CSharp):

1. Shader "Custom/Slice"
2. {
3.     Properties
4.     {
5.         _Color("Color", Color) = (1,1,1,1)
6.         _MainTex("Albedo (RGB)", 2D) = "white" {}
7.         _Glossiness("Smoothness", Range(0,1)) = 0.5
8.         _Metallic("Metallic", Range(0,1)) = 0.0
9.         [HDR]_Emission("Emission", Color) = (1,1,1,1)
10.         _NoiseSize("Noise size", float) = 1
11.     }
12.     SubShader
13.     {
14.         Tags{ "RenderType" = "Opaque" }
15.         Cull off
16.         LOD 200
17.  
18.         CGPROGRAM
19.         // Physically based Standard lighting model, and enable shadows on all light types
20.         #pragma surface surf Standard fullforwardshadows addshadow
21.      
22.         // Use shader model 3.0 target, to get nicer looking lighting
23.         #pragma target 3.0
24.  
25.         sampler2D _MainTex;
26.  
27.         struct Input
28.         {
29.             float2 uv_MainTex;
30.             float3 worldPos;
31.         };
32.  
33.         half _Glossiness;
34.         half _Metallic;
35.         fixed4 _Color;
36.  
37.         fixed4 _Emission;
38.         float _NoiseSize;
39.  
40.         float4 _GLOBALMaskPosition;
41.         half _GLOBALMaskRadius;
42.         half _GLOBALMaskSoftness;
43.  
44.         // Add instancing support for this shader. You need to check 'Enable Instancing' on materials that use the shader.
45.         // See https://docs.unity3d.com/Manual/GPUInstancing.html for more information about instancing.
46.         // #pragma instancing_options assumeuniformscaling
47.         UNITY_INSTANCING_BUFFER_START(Props)
48.         // put more per-instance properties here
49.         UNITY_INSTANCING_BUFFER_END(Props)
50.  
51.         float random(float2 input)
52.         {
53.             return frac(sin(dot(input, float2(12.9898,78.233)))* 43758.5453123);
54.         }
55.  
56.         void surf(Input IN, inout SurfaceOutputStandard o)
57.         {
58.             fixed4 c = tex2D(_MainTex, IN.uv_MainTex) * _Color;
59.             half dist = distance(_GLOBALMaskPosition, IN.worldPos);
60.             half sphere = 1 - saturate((dist - _GLOBALMaskRadius) / _GLOBALMaskSoftness);
61.             clip(sphere - 0.1);
62.          
63.             float squares = step(0.5, random(floor(IN.uv_MainTex * _NoiseSize)));
64.             half emissionRing = step(sphere - 0.1, 0.1) * squares;
65.             o.Emission = _Emission * emissionRing;
66.             o.Albedo = c.rgb;
67.             o.Metallic = _Metallic;
68.             o.Smoothness = _Glossiness;
69.             o.Alpha = c.a;
70.         }
71.     ENDCG
72.     }
73.     FallBack "Diffuse"
74.     //Fallback "Legacy Shaders/Transparent/Cutout/VertexLit"
75. }

Then the script that would be your geometry that would act as your mask would look like this.

Code (CSharp):

1. using System.Collections;
2. using System.Collections.Generic;
3. using UnityEngine;
4.  
5. [ExecuteInEditMode]
6. public class SphereMaskController : MonoBehaviour
7. {
8.  
9.     public float radius = 0.5f;
10.     public float softness = 0.5f;
11.  
12.     void Update()
13.     {
14.         Shader.SetGlobalVector("_GLOBALMaskPosition", transform.position);
15.         Shader.SetGlobalFloat("_GLOBALMaskRadius", radius);
16.         Shader.SetGlobalFloat("_GLOBALMaskSoftness", softness);
17.     }
18. }

 

This is making the same mistake I mentioned above. The mask position is a float4 and the world position is a float3. On some platforms this will produce unwanted results, and on most others it won't compile at all. Basically, this works on Windows when compiling for Direct3D and nothing else because the Direct3D shader compiler makes some assumptions and fixes the bad code for you, where as other platforms will just tell you that's wrong. Just define _GLOBALMaskPosition as a float3 to avoid this.

 

Right! float3 for XYZ location. Got it thank you!

Would I want to use a float3 when using SV_POSITION?

 

SV_POSITION is the clip space position, which is a float4. It's not a simple 3d position.

 

I have a problem similar to yours, can you help me?

The idea is the same, I have a mesh and it should appear on a table. The difference is that the mask shader is rectangular and not circular.

I'm new to Unity. I tried to reproduce your tutorial but it did not work for me, I have some doubts.

I have the rectangle that will be the shader and the mockup, I added your shader to the cube and then the script too. I do not know what I do after that, the cube is transparent but does not work (cuts) with the model.

Could you help me make the rectangular shader and how to properly apply to the cube so that I have the effect like yours?