Color mask shader

Hello everybody. I need a shader that applies a color mask to the image and accepts an array of colors to filter out the mask. Now I have this:

Shader "Custom/Test"

{

    Properties

    {

        [PerRendererData] _MainTex("Sprite Texture", 2D) = "white" {}

        [PerRendererData] _MaskTex("Mask Texture", 2D) = "white" {}

        _Color("Tint", Color) = (1,1,1,1)

        _DefaultColor("Default Color", Color) = (1,1,1,1)

        _Accuracy("Accuracy", Range(0, 2)) = 1.1

        [MaterialToggle] _ShowMaskImage("Show Mask Image", Float) = 0

    }

        SubShader

        {

            Tags

            {

                "Queue" = "Transparent"

                "IgnoreProjector" = "True"

                "RenderType" = "Transparent"

                "PreviewType" = "Plane"

                "CanUseSpriteAtlas" = "True"

            }



            Cull Off

            Lighting Off

            ZWrite Off

            Blend One OneMinusSrcAlpha



            Pass

            {

            CGPROGRAM

                #pragma vertex vert

                #pragma fragment frag

                #include "UnityCG.cginc"



                struct appdata_t

                {

                    float4 vertex   : POSITION;

                    float4 color    : COLOR;

                    float2 texcoord : TEXCOORD0;

                };



                struct v2f

                {

                    float4 vertex   : SV_POSITION;

                    fixed4 color : COLOR;

                    float2 texcoord  : TEXCOORD0;

                };



                fixed4 _Color;



                v2f vert(appdata_t IN)

                {

                    v2f OUT;

                    OUT.vertex = UnityObjectToClipPos(IN.vertex);

                    OUT.texcoord = IN.texcoord;

                    OUT.color = IN.color * _Color;



                    return OUT;

                }



                fixed3 _MaskColors[32];

                int _MaskColorsCount = 0;

                fixed4 _DefaultColor;

                fixed _Accuracy;

                bool _ShowMaskImage;

                sampler2D _MainTex;

                sampler2D _MaskTex;



                fixed4 frag(v2f IN) : SV_Target

                {

                    fixed4 res = tex2D(_MainTex, IN.texcoord) * IN.color;

                    fixed4 mask = tex2D(_MaskTex, IN.texcoord);



                    for (int i = 0; i < _MaskColorsCount; i++)

                    {

                        fixed3 diff = abs(mask.rgb - _MaskColors[i].rgb);

                        fixed sum = diff.r + diff.g + diff.b;



                        mask.a = sum < _Accuracy ? 0 : mask.a;

                    }



                    mask.rgb = _ShowMaskImage ? mask.rgb : _DefaultColor;



                    res.rgb *= res.a;

                    mask.rgb *= mask.a;

                    return lerp(res, mask, mask.a);

                }

            ENDCG

            }

        }

}

But there is a drawback, the boundaries between the mask and the image are sharp.


I tried to change the shader for a smoother transition:

Shader "Custom/Test2"

{

    Properties

    {

        [PerRendererData] _MainTex("Sprite Texture", 2D) = "white" {}

        [PerRendererData] _MaskTex("Mask Texture", 2D) = "white" {}

        _Color("Tint", Color) = (1,1,1,1)

        _DefaultColor("Default Color", Color) = (1,1,1,1)

        _Accuracy("Accuracy", Range(0, 2)) = 1.1

        [MaterialToggle] _ShowMaskImage("Show Mask Image", Float) = 0

    }

    SubShader

    {

        Tags

        { 

            "Queue"="Transparent" 

            "IgnoreProjector"="True" 

            "RenderType"="Transparent" 

            "PreviewType"="Plane"

            "CanUseSpriteAtlas"="True"

        }



        Cull Off

        Lighting Off

        ZWrite Off

        Blend One OneMinusSrcAlpha



        Pass

        {

        CGPROGRAM

            #pragma vertex vert

            #pragma fragment frag

            #pragma enable_d3d11_debug_symbols

            #include "UnityCG.cginc"

           

            struct appdata_t

            {

                float4 vertex   : POSITION;

                float4 color    : COLOR;

                float2 texcoord : TEXCOORD0;

            };



            struct v2f

            {

                float4 vertex   : SV_POSITION;

                fixed4 color    : COLOR;

                float2 texcoord  : TEXCOORD0;

            };

           

            fixed4 _Color;



            v2f vert(appdata_t IN)

            {

                v2f OUT;

                OUT.vertex = UnityObjectToClipPos(IN.vertex);

                OUT.texcoord = IN.texcoord;

                OUT.color = IN.color * _Color;



                return OUT;

            }



            fixed3 _MaskColors[512];

            int _MaskColorsCount = 0;

            fixed4 _DefaultColor;

            fixed _Accuracy;

            bool _ShowMaskImage;

            sampler2D _MainTex;

            sampler2D _MaskTex;



            fixed4 frag(v2f IN) : SV_Target

            {

                fixed4 res = tex2D(_MainTex, IN.texcoord) * IN.color;

                fixed4 mask = tex2D(_MaskTex, IN.texcoord);

               

                fixed maskAlpha = mask.a;

                mask.a = 0;



                bool changed = false;

                for (int i = 0; i < _MaskColorsCount; i++)

                {

                    fixed3 diff = abs(mask.rgb - _MaskColors[i].rgb);

                    fixed sum = diff.r + diff.g + diff.b;



                    changed = sum < _Accuracy ? true : changed;

                    mask.a += sum < _Accuracy ? sum : 0;

                }



                mask.a = changed ? clamp(mask.a, 0, 1) : maskAlpha;



                mask.rgb = _ShowMaskImage ? mask.rgb : _DefaultColor;



                res.rgb *= res.a;

                mask.rgb *= mask.a;

                return lerp(res, mask, mask.a);

            }

        ENDCG

        }

    }

}

But if two colors next to each other are deselected, the transition is visible between them.


What am I doing wrong? How can the shader be improved? Or is it better to write a new one differently?

P.S. My test mask