Search Unity

Problems with writing a cutout shader, that can recieve shadows

Discussion in 'Shaders' started by FabioZocklabs, Jul 11, 2019.

  1. FabioZocklabs

    FabioZocklabs

    Joined:
    Jul 11, 2019
    Posts:
    17
    eI tried multiple approaches, but non of them worked.

    Here a try with two shader passes:

    ```
    Shader "Cutout" {

    Properties {
    _MainTex ("Base (RGB)", 2D) = "white" {}
    _Tint ("Tint", Color) = (1, 1, 1, 1)
    _Brightness ("Brightness", Range(0,4)) = 1.25
    _Light ("Light", Color) = (0, 0, 0, 1)
    _Shadows ("Shadows", Range(0,1)) = 0.75
    _Contrast ("Contrast", Range(-1,1)) = 0.0
    _Cutoff ("Alpha cutoff", Range(0,1)) = 0.5
    }

    SubShader {
    Tags {
    "RenderType"="TransparentCutout"
    }
    LOD 80
    Pass {
    Name "Default"
    CGPROGRAM
    #pragma vertex vert
    #pragma fragment frag
    #pragma multi_compile_fwdbase
    #include "UnityCG.cginc"
    #include "Lighting.cginc"
    #pragma multi_compile_fwdbase
    #include "AutoLight.cginc"
    sampler2D _MainTex;
    float4 _MainTex_ST;
    float4 _Tint;
    fixed _Brightness;
    fixed4 _Light;
    fixed _Contrast;
    fixed _Cutoff;
    struct v2f {
    float4 pos : SV_POSITION;
    float2 texcoord : TEXCOORD1;
    float3 normal : NORMAL;
    };
    v2f vert(appdata_base v) {
    v2f o;
    o.pos = UnityObjectToClipPos (v.vertex);
    o.texcoord = TRANSFORM_TEX(v.texcoord, _MainTex);
    o.normal = normalize(mul(float4(v.normal, 0.0), unity_WorldToObject).xyz);

    return o;
    }
    fixed4 frag(v2f i) : SV_TARGET {

    float4x4 modelMatrix = unity_ObjectToWorld;

    float3 normalDirection = normalize(i.normal);

    float4 col = tex2D(_MainTex, i.texcoord) * _Tint;
    if (col.a < _Cutoff) discard;

    float3 lightColor = _LightColor0.rgb * _Brightness;
    float3 greyCol = (col.r + col.g + col.b) / 3.0f;
    float3 realCol = lerp(col, greyCol, -_Contrast);

    return fixed4(realCol.xyz * lightColor + _Light.xyz, 1.0);

    }
    ENDCG
    }
    Pass {
    Blend SrcAlpha OneMinusSrcAlpha
    Name "ShadowCatcher"
    Tags { "LightMode" = "ForwardBase" }
    CGPROGRAM
    #pragma vertex vert
    #pragma fragment frag Standard fullforwardshadows alphatest:_Cutoff addshadow
    #pragma multi_compile_fwdbase
    #include "UnityCG.cginc"
    #include "Lighting.cginc"
    #pragma multi_compile_fwdbase
    #include "AutoLight.cginc"
    sampler2D _MainTex;
    float4 _MainTex_ST;
    fixed _Shadows;
    fixed _Cutoff;
    struct v2f {
    float4 pos : SV_POSITION;
    float2 texcoord : TEXCOORD1;
    LIGHTING_COORDS(2,3)
    };
    v2f vert(appdata_base v) {
    v2f o;
    o.pos = UnityObjectToClipPos (v.vertex);
    TRANSFER_VERTEX_TO_FRAGMENT(o);
    o.texcoord = TRANSFORM_TEX(v.texcoord, _MainTex);
    return o;
    }
    fixed4 frag(v2f i) : SV_TARGET {

    float atten = LIGHT_ATTENUATION(i);
    half4 c;
    c.rgb = 0;
    c.a = (1 - atten) * _Shadows;
    if (tex2D(_MainTex, i.texcoord).a < _Cutoff) c.a = 0;
    return c;
    }
    ENDCG
    }
    }

    }
    ```
    Here one with only one:

    ```
    Shader "Cutout" {

    Properties {
    _MainTex ("Base (RGB)", 2D) = "white" {}
    _Tint ("Tint", Color) = (1, 1, 1, 1)
    _Brightness ("Brightness", Range(0,4)) = 1.25
    _Light ("Light", Color) = (0, 0, 0, 1)
    _Shadows ("Shadows", Range(0,1)) = 0.75
    _Contrast ("Contrast", Range(-1,1)) = 0.0
    _Cutoff ("Alpha cutoff", Range(0,1)) = 0.5
    }

    SubShader {
    Tags {
    "RenderType"="TransparentCutout"
    "Queue"="AlphaTest"
    "IgnoreProjector"="True"
    }
    LOD 80
    Pass {
    Name "ShadowCatcher"
    Tags { "LightMode" = "ForwardBase" }
    CGPROGRAM
    #pragma vertex vert
    #pragma fragment frag
    #pragma multi_compile_fwdbase
    #include "UnityCG.cginc"
    #include "Lighting.cginc"
    #pragma multi_compile_fwdbase
    #include "AutoLight.cginc"
    sampler2D _MainTex;
    float4 _MainTex_ST;
    float4 _Tint;
    fixed _Brightness;
    fixed _Shadows;
    fixed4 _Light;
    fixed _Contrast;
    fixed _Cutoff;
    struct v2f {
    float4 pos : SV_POSITION;
    float2 texcoord : TEXCOORD1;
    float3 normal : NORMAL;
    LIGHTING_COORDS(2,3)
    };
    v2f vert(appdata_base v) {
    v2f o;
    o.pos = UnityObjectToClipPos (v.vertex);
    TRANSFER_VERTEX_TO_FRAGMENT(o);
    o.texcoord = TRANSFORM_TEX(v.texcoord, _MainTex);
    o.normal = normalize(mul(float4(v.normal, 0.0), unity_WorldToObject).xyz);

    return o;
    }
    fixed4 frag(v2f i) : SV_TARGET {

    float4x4 modelMatrix = unity_ObjectToWorld;

    float3 normalDirection = normalize(i.normal);

    float4 col = tex2D(_MainTex, i.texcoord) * _Tint;
    if (col.a < _Cutoff) discard;

    float shadowFac =
    _Brightness *
    (1.0 - (1.0 - LIGHT_ATTENUATION(i)) * _Shadows);

    float3 lightColor = _LightColor0.rgb * shadowFac;
    float3 greyCol = (col.r + col.g + col.b) / 3.0f;
    float3 realCol = lerp(col, greyCol, -_Contrast);

    return fixed4(realCol.xyz * lightColor + _Light.xyz, 1.0);

    }
    ENDCG
    }

    }

    }
    ```

    Both results look the same; in both cases I don't see the shadows thrown onto the object, but the shadows thrown onto objects hidden by the object with this shader. This is what I would expect for the transparent parts.

    If I add a custom shadow caster pass, which does never discard the result, everything catches the shadow, even the transparent parts...

    Any idea how to fix this?
     
  2. FabioZocklabs

    FabioZocklabs

    Joined:
    Jul 11, 2019
    Posts:
    17
    I finally found my problem. The custom shadow caster pass was wrong.