Search Unity

  1. Unity 2019.2 is now released.
    Dismiss Notice

Trying to create stencil lights but I am hopelessly lost

Discussion in 'Shaders' started by Quakeulf, Oct 28, 2018.

  1. Quakeulf

    Quakeulf

    Joined:
    Mar 26, 2013
    Posts:
    40
    Hi,
    I have looked all over the internets for this, but found nothing that could help me get anywhere. Reading the documentation made me realise all there is left for me to do is read up on everything and just try until I have something that works. There are no assets on the asset store, or code available that I could find anywhere either. All I could find that was relevant to this was this post, but it had no code to look at: https://polycount.com/discussion/185838/wind-waker-style-firefly-lights

    So the question is, how can I make this work with stencils in shaders in Unity 3D? Attached is the current setup and hilited in blue is the overlapping issues I get (among others). I started this based on the Unity Stencil documentation here: https://docs.unity3d.com/Manual/SL-Stencil.html

    The shader code I have is here:
    Code (CSharp):
    1. Shader "Test/Light" {
    2.     Properties {
    3.         _Color ("Main Color", Color) = (1,1,1,0)
    4.     }
    5.     SubShader {
    6.         Tags { "RenderType"="Opaque" "Queue"="Geometry+2"}
    7.  
    8.         ColorMask RGB
    9.         Cull Front
    10.         ZTest Always
    11.         Blend One One
    12.         Stencil {
    13.             Ref 1
    14.             Comp notequal
    15.         }
    16.  
    17. Pass
    18.         {
    19.             CGPROGRAM
    20.             #pragma vertex vert
    21.             #pragma fragment frag
    22.             // make fog work
    23.             #pragma multi_compile_fog
    24.          
    25.             #include "UnityCG.cginc"
    26.  
    27.             fixed4 _Color;
    28.  
    29.             struct appdata
    30.             {
    31.                 float4 vertex : POSITION;
    32.             };
    33.  
    34.             struct v2f
    35.             {
    36.                 float4 vertex : SV_POSITION;
    37.             };
    38.          
    39.             v2f vert (appdata v)
    40.             {
    41.                 v2f o;
    42.                 o.vertex = UnityObjectToClipPos(float4(v.vertex.xyz, 1.0f)); //UnityObjectToClipPos(v.vertex);
    43.                 return o;
    44.             }
    45.          
    46.             fixed4 frag () : SV_Target
    47.             {
    48.                 return _Color;
    49.             }
    50.             ENDCG
    51.         }
    52.     }
    53. }
    And here:

    Code (CSharp):
    1. Shader "Test/LightHelp" {
    2.     SubShader {
    3.         Tags { "RenderType"="Opaque" "Queue"="Geometry+1"}
    4.         ColorMask 0
    5.         ZWrite off
    6.         Stencil {
    7.             Ref 1
    8.             Comp always
    9.             Pass replace
    10.         }
    11.  
    12.         CGINCLUDE
    13.             struct appdata {
    14.                 float4 vertex : POSITION;
    15.             };
    16.             struct v2f {
    17.                 float4 pos : SV_POSITION;
    18.             };
    19.             v2f vert(appdata v) {
    20.                 v2f o;
    21.                 o.pos = UnityObjectToClipPos(v.vertex);
    22.                 return o;
    23.             }
    24.             half4 frag(v2f i) : SV_Target {
    25.                 return half4(1,1,0,1);
    26.             }
    27.         ENDCG
    28.  
    29.         Pass {
    30.             Cull Front
    31.             ZTest Less
    32.      
    33.             CGPROGRAM
    34.             #pragma vertex vert
    35.             #pragma fragment frag
    36.             ENDCG
    37.         }
    38.         Pass {
    39.             Cull Back
    40.             ZTest Greater
    41.      
    42.             CGPROGRAM
    43.             #pragma vertex vert
    44.             #pragma fragment frag
    45.             ENDCG
    46.         }
    47.     }
    48. }
    All I want is for the surface that intersects the mesh to be coloured in a bright colour without the issues you can see in the attached image "show2.PNG" below, but I seem to be unable to get somewhere with this.

    Please help. (There is also an asset store idea for you here as mentioned earlier, I'd be willing to pay for it and am sure others would too.)
     

    Attached Files:

    ParametricAvocado likes this.
  2. ParametricAvocado

    ParametricAvocado

    Joined:
    Sep 19, 2013
    Posts:
    3
    Hey there!

    I saw your tweet and, as I said over there, the issue is tied to multiple aspects of how your shaders were set up.
    Since the "Light" shader only draws fragments wherever neither of the stencil passes have before, but doesn't clear the stencil buffer afterwards, what's happening is the Second Stencil Pass from the light cone further away is writing 1 to the stencil Buffer before the Light pass of the light cone closest to the camera is drawn.
    The quick fix to this is to include Pass, Fail and ZFail all set to Zero in the "Light" shader's Stencil block, this way all the stencil buffer values changed by each light cone should reset right before the next one begins to draw.

    I'd also recommend making these kinds of shaders' RenderType and Queue to "Transparent" and "Transparent-10" respectively and have them not write to the Zbuffer (Zwrite Off) so that they don't get in the way of other Geometry-typed shaders and also don't mess with the depth information so that other visuals such as transparent particles behave as expected and always draw on top instead of inconsistently flickering due to sorting issues.

    I've expanded a little bit on the idea, making a quick single-material, 2-pass Stencil Light shader that allows additive light volume overlapping so that it essentially behaves like normal lights minus falloff (that requires some more in-depth Zbuffer handling, pun intended).

    Check these out:



    Code (CSharp):
    1. Shader "Custom/StencilLight"
    2. {
    3.     Properties
    4.     {
    5.         _MainTex("Texture", 2D) = "white" {}
    6.         _Color("Color",Color) = (1,1,1,0.5)
    7.  
    8.         _StencilRef("Stencil Reference Value", Float) = 20
    9.     }
    10.     SubShader
    11.     {
    12.         Tags { "RenderType" = "Transparent" "Queue" = "Transparent-10"}
    13.         LOD 100
    14.         Pass
    15.         {
    16.             //This pass only draws under opaque objects
    17.             Ztest Greater
    18.             //Does not affect the zbuffer, we will still need surface depth information
    19.             //not only for this shader but for the other transparent stuff that might get drawn after
    20.             Zwrite off
    21.             //Draw the insides of the primitive
    22.             Cull Front
    23.             //Do not output any color information
    24.             Colormask 0
    25.             Name "Stencil Greater"
    26.             //As long as the Ztest passes, 20 shall be written to the stencil buffer
    27.             Stencil
    28.             {
    29.                 comp always
    30.                 ref [_StencilRef]
    31.                 pass replace
    32.             }
    33.         }
    34.         /*
    35.         This pass is now redundant
    36.         Pass
    37.         {
    38.             Zwrite off
    39.             Ztest lequal
    40.             Cull Back
    41.             Colormask 0
    42.             Name "Stencil LEqual"
    43.             Stencil
    44.             {
    45.                 comp equal
    46.                 ref 20
    47.                 pass keep
    48.                 fail zero
    49.                 zfail zero
    50.             }
    51.         }*/
    52.  
    53.         Pass
    54.         {
    55.             Name "Color"
    56.             //Once more we don't want to touch the zbuffer
    57.             Zwrite off
    58.             //Standard Ztest and backface culling, left these here just for clarity
    59.             Ztest Lequal
    60.             Cull Back
    61.             //Additive, but taking alpha output into account for blending.
    62.             Blend SrcAlpha One
    63.  
    64.             //Will only draw if intersecting with the stencil value from the previous pass
    65.             //Even if the Ztest fails, it will still clear the stencil value
    66.             //so that any following lights also render correctly without reading previous lights' stencil values.
    67.             Stencil
    68.             {
    69.                 comp equal
    70.                 ref [_StencilRef]
    71.                 pass zero
    72.                 fail zero
    73.                 zfail zero
    74.             }
    75.  
    76.             CGPROGRAM
    77.             #pragma vertex vert
    78.             #pragma fragment frag
    79.  
    80.             #include "UnityCG.cginc"
    81.  
    82.             struct appdata
    83.             {
    84.                 float4 vertex : POSITION;
    85.                 float2 uv : TEXCOORD0;
    86.             };
    87.  
    88.             struct v2f
    89.             {
    90.                 float2 uv : TEXCOORD0;
    91.                 float4 vertex : SV_POSITION;
    92.             };
    93.  
    94.             sampler2D _MainTex;
    95.             float4 _MainTex_ST;
    96.             fixed4 _Color;
    97.             v2f vert(appdata v)
    98.             {
    99.                 v2f o;
    100.                 o.vertex = UnityObjectToClipPos(v.vertex);
    101.                 o.uv = TRANSFORM_TEX(v.uv, _MainTex);
    102.                 return o;
    103.             }
    104.  
    105.             fixed4 frag(v2f i) : SV_Target
    106.             {
    107.                 fixed4 col = tex2D(_MainTex, i.uv) *_Color;
    108.                 return col;
    109.             }
    110.             ENDCG
    111.         }
    112.     }
    113. }
    Be sure to notice multi-pass shader drawcalls are not dynamically batched, and at least for this approach, that is good, because after each stencil-only pass, a "Color" pass is needed in order to clear the Stencil Buffer before the next light volume is drawn so that no weird stencil overlaps happen. This amounts to 2 drawcalls for each light volume, which can get a bit expensive so use with care.



    I hope this helps you on your journey to better, less glitchy streetlights. Have fun!
     
  3. Quakeulf

    Quakeulf

    Joined:
    Mar 26, 2013
    Posts:
    40
    Thank you so much for this! I will try it out once I get home from work today. :3

    I am fullly aware of the drawcalls needed, but I thought maybe I could take a CombineMesh on the streetlight volumes per "block" or per street, and that could help get the drawcalls down. I am not that concerned about that, more about getting it to work for the intended effect, and your explanation is great for this. I think maybe I can get the falloff working because I have an idea how that might work (similar to what I did for the windows, except it takes distance from object center into consideration instead of cameraposition in relation to object world position).
     
    ParametricAvocado likes this.
  4. Quakeulf

    Quakeulf

    Joined:
    Mar 26, 2013
    Posts:
    40
    Oh by the way, what would it take to make the intersection only valid for the first object surface facing towards the center of the light? In its current configuration the intersectional light also works for faces that face away from it, and that also are behind other faces. I think that doing a normal-check might help solve it, but I am not sure exactly how.
     
    ParametricAvocado likes this.
  5. ParametricAvocado

    ParametricAvocado

    Joined:
    Sep 19, 2013
    Posts:
    3
    To answer to the first message, while combining all meshes would reduce drawcalls, it'd mean a single pass would write the stencil for many light volumes at the same time, that would cause the Color pass to maybe sometimes draw one light volume over another's stencil mask, creating artifacts and plain wrong intersections.

    Making the lights behave more like normal lights would require you to sample both the depth and normals texture each frame in order to obtain the surface information, since the part that is rendered in the Color pass is nothing like the surface it seems to be projected onto. That'd be much closer to the usual Deferred Lighting approach since that's the main purpose of it's G-buffer.

    On top of that, making the lights stop at the first object they hit would probably require depth textures, or a more complex stencil system paired with vertex extrusion techniques applied to at least the dynamic objects involved.

    If you want the lights to only intersect with static, simpler geometry, maybe you could get away with raycasting each extruded vertex against, say, the ground mesh, then add a little offset to make the light volume mesh actually intersect the surface, in Editor time.
     
  6. Quakeulf

    Quakeulf

    Joined:
    Mar 26, 2013
    Posts:
    40
    Well, this maybe is a little more complex than I thought, then... ;_;

    But also, looking at the blending that happens here it looks like the blending the lights currently have is completely different: https://agilethief.artstation.com/projects/OBBXy?album_id=70137

    What is the secret to this blending? This is exactly what I would like to have. Is it to render everything in the shade and then "cut out" from that shaded area?
     
  7. ParametricAvocado

    ParametricAvocado

    Joined:
    Sep 19, 2013
    Posts:
    3
    Hello and sorry for the delayed reply.

    The effect shown in that example is achieved by using Additive blending operations during the Color Pass(which is the same thing my shader does), apparently on top of standard lighting, which is also rendered additively. Notice the smooth lighting on the character, which doesn't seem to be affected by the torch, presumably because the character mesh is also writing to the stencil buffer in order to avoid being lit by the stencil lights. Either that or the character's draw calls are queued after all opaque environment and stencil lights are rendered.

    In a Deferred Lighting scenario, the surface's Glossiness, Albedo, Normal and AO information could also be read to further modify how the lit fragments behave.

    I went ahead and set up a simple scene using the same shader I posted a few days back.

    In order to achieve such a noticeable effect you should avoid reaching extreme lighting values and create contrast by supplying lower ambient lighting values. Having the scenario meshes be lit by the standard light is optional and entirely an aesthetic choice.

    In this scene I completely turned off sunlight, thus making the default procedural sky darken and "naturally" lowering the real time ambient light in the scene and giving it a rather blueish tint that makes the warmth of the fire even more noticeable. On top of that, I could've used different meshes instead of Unity's default UV Sphere to achieve the polygonal look in agilethief's example.
     
  8. Quakeulf

    Quakeulf

    Joined:
    Mar 26, 2013
    Posts:
    40
    I tried turning off the sunlight, using #414141 for the light colour, switching to deferred lighting, and using 0.5 ambient skylight, and this is what it ended up looking like against objects using the Standard shader.

    It does not look like in your example.

    Am I not understanding, are extra lights needed (as in the campfire) to supplement this, or does this not work in WebGL?
     

    Attached Files:

  9. Nyanpas

    Nyanpas

    Joined:
    Dec 29, 2016
    Posts:
    39
    Code (CSharp):
    1. Shader "Custom/StencilLight"
    2. {
    3.     Properties
    4.     {
    5.         _Color("Color",Color) = (1,1,1,0.5)
    6.         _Intensity("Intensity", Float) = 1
    7.  
    8.         _StencilRef("Stencil Reference Value", Float) = 20
    9.     }
    10.  
    11.     SubShader
    12.     {
    13.         Tags
    14.         {
    15.             "RenderType" = "Transparent"
    16.             "Queue" = "Transparent"
    17.         }
    18.  
    19.         LOD 100
    20.  
    21.         Pass
    22.         {
    23.             Name "Stencil Greater"
    24.             Ztest Greater
    25.             Zwrite off
    26.             Cull Off
    27.             Colormask 0
    28.             Lighting Off
    29.  
    30.             Stencil
    31.             {
    32.                 comp always
    33.                 ref[_StencilRef]
    34.                 pass replace
    35.             }
    36.         }
    37.  
    38.         Pass
    39.         {
    40.             Name "Color"
    41.             Zwrite off
    42.             Ztest Lequal
    43.             Cull Back
    44.             Lighting Off
    45.             Blend SrcAlpha One
    46.  
    47.             Stencil
    48.             {
    49.                 comp equal
    50.                 ref[_StencilRef]
    51.                 pass zero
    52.                 fail zero
    53.                 zfail zero
    54.             }
    55.  
    56.             CGPROGRAM
    57.             #pragma vertex vert
    58.             #pragma fragment frag
    59.  
    60.             fixed _Intensity;
    61.  
    62.             struct appdata
    63.             {
    64.                 float4 vertex : POSITION;
    65.             };
    66.  
    67.             struct v2f
    68.             {
    69.                 float4 vertex : SV_POSITION;
    70.             };
    71.  
    72.             fixed4 _Color;
    73.             v2f vert(appdata v)
    74.             {
    75.                 v2f o;
    76.                 o.vertex = UnityObjectToClipPos(v.vertex);
    77.                 return o;
    78.             }
    79.  
    80.             fixed4 frag(v2f i) : SV_Target
    81.             {
    82.                 return _Color * _Intensity;
    83.             }
    84.  
    85.             ENDCG
    86.         }
    87.  
    88.         Pass
    89.         {
    90.             Name "Color 2"
    91.             ZTest off
    92.             ZWrite on
    93.             Cull Front
    94.             Lighting Off
    95.             Blend SrcAlpha One
    96.             BlendOp Add
    97.  
    98.             Stencil
    99.             {
    100.                 Ref[_StencilRef]
    101.                 Comp equal
    102.                 Pass zero
    103.             }
    104.  
    105.             CGPROGRAM
    106.             #pragma vertex vert
    107.             #pragma fragment frag
    108.  
    109.             fixed _Intensity;
    110.  
    111.             struct appdata
    112.             {
    113.                 float4 vertex : POSITION;
    114.             };
    115.  
    116.             struct v2f
    117.             {
    118.                 float4 vertex : SV_POSITION;
    119.             };
    120.  
    121.             fixed4 _Color;
    122.             v2f vert(appdata v)
    123.             {
    124.                 v2f o;
    125.                 o.vertex = UnityObjectToClipPos(v.vertex);
    126.                 return o;
    127.             }
    128.  
    129.             fixed4 frag(v2f i) : SV_Target
    130.             {
    131.                 return _Color * _Intensity;
    132.             }
    133.  
    134.             ENDCG
    135.         }
    136.     }
    137. }
    Here is adding a third pass so that I can go inside the 3D object without having it disappear on me, however I would like to get it down to just two passes if possible.