Wall color Detection and painting algorithm

Hi guys,

I am working on an app to detect wall colour and then selecting a custom colour from the UI and applying on it.
At the moment i have a fragment shader which takes gives a noisy output.
I also tried using Hue Sat Value but its not giving a proper output.
Its just giving a grainy colour by coloring some pixels.

I am clicking on a point and grabbing the color and then passing that using a cs script to my fragment shader.

Code (CSharp):

1.  
2. void Start()
3.     {
4.         // Start web cam feed
5.         webcamTexture = new WebCamTexture();
6.  
7.         VidPlane.GetComponent<Renderer>().material.mainTexture = webcamTexture;
8.  
9.         webcamTexture.Play();    
10.     }
11.  
12.     int i = 0, j = 0, s = 0;
13.     Color allcols;
14.  
15.  
16.     void Update()
17.     {
18.         if (Input.GetMouseButtonDown(0))
19.         {
20.             RaycastHit hit = new RaycastHit();
21.  
22.             Ray ray = Camera.main.ScreenPointToRay(Input.mousePosition);
23.             if (Physics.Raycast(ray, out hit))
24.             {
25.                 float H1, S1, V1;
26.                 GameObject hitobj = hit.transform.gameObject;
27.                 Vector3 screenPos = cam.WorldToScreenPoint(hit.point);
28.  
29.                 // map coordinates to display resolution (e. g. 1920 x 1080)
30.                 Debug.Log("Click position relative to the screen size (e. g. 2560 x 1080) " + screenPos);
31.  
32.                 Vector2 screenRatio = new Vector2(screenPos.x / Screen.width, screenPos.y / Screen.height);
33.                 Color color = webcamTexture.GetPixel((int)(screenRatio.x * webcamTexture.width), (int)(screenRatio.y * webcamTexture.height));
34.  
35.             //    Color color = webcamTexture.GetPixel((int)(hit.point.x), (int)(hit.point.x));
36.                 Debug.Log(hit.point);
37.                 clickImg.color = color;
38.                 Color.RGBToHSV(color, out H1, out S1, out V1);
39.              
40.                 matTex.GetComponent<Renderer>().material.SetColor("_Color", color);
41.                 matTex.GetComponent<Renderer>().material.SetFloat("_Hue1", H1);
42.                 matTex.GetComponent<Renderer>().material.SetFloat("_Sat1", S1);
43.                 matTex.GetComponent<Renderer>().material.SetFloat("_Val1", V1);
44.  
45.                 Debug.Log("Color x/y: " + color);
46.                 matTex.GetComponent<Renderer>().material.color = color;
47.             }
48.         }
49. }
50.  

Shader "Custom/WallColorShader" {

    Properties{

        _Color("Color", Color) = (1,1,1,1)

        _MainTex("Albedo (RGB)", 2D) = "white" {}

        _SecondaryTex("Secondary Texture", 2D) = "white" {}

        _Threshold("Threshold", Range(0, 1)) = 0

        _ColorToReplace("Color To Replace", Color) = (1.0, 0.0, 0.0, 1.0)

        _Hue1("Hue", Float) = 0                // Not using

        _Sat1("Saturation", Float) = 1

        _Val1("Value", Float) = 1

    }

        SubShader{

            Pass {

                Tags { "RenderType" = "Opaque" }



                CGPROGRAM

                #pragma vertex vert

                #pragma fragment frag

                #include "UnityCG.cginc"



                struct v2f {

                    float4 pos : SV_POSITION;

                    float2 uv1 : TEXCOORD0;

                    float2 uv2 : TEXCOORD1;

                };



                sampler2D _MainTex;

                float4 _MainTex_ST;

                sampler2D _SecondaryTex;

                float4 _SecondaryTex_ST;

                uniform fixed4 _ColorToReplace;

                fixed4 _Color;



                v2f vert(appdata_base v) {

                    v2f o;

                    o.pos = UnityObjectToClipPos(v.vertex);

                    o.uv1 = TRANSFORM_TEX(v.texcoord, _MainTex);

                    o.uv2 = TRANSFORM_TEX(v.texcoord, _SecondaryTex);

                    return o;

                }



                float _Threshold;



                fixed4 frag(v2f i) : COLOR {

                    fixed4 col1 = tex2D(_MainTex, i.uv1);

                    fixed4 col2 = tex2D(_SecondaryTex, i.uv2);

                    fixed4 valb = ceil(saturate(col1.b - col1.r - _Threshold)) * ceil(saturate(col1.b - col1.g - _Threshold));

                    fixed4 valg = ceil(saturate(col1.g - col1.r - _Threshold)) * ceil(saturate(col1.g - col1.b - _Threshold));

                    fixed4 valr = ceil(saturate(col1.r - col1.g - _Threshold)) * ceil(saturate(col1.r - col1.b - _Threshold));



                    if ((_Color.b > _Color.r) && (_Color.b > _Color.g))

                    {

                            return lerp(col1, col2, valb);

                    }

                    else if ((_Color.g > _Color.r) && (_Color.g > _Color.b))

                    {

                            return lerp(col1, col2, valg);

                    }

                    else if ((_Color.r > _Color.g) && (_Color.r > _Color.b))

                    {

                            return lerp(col1, col2, valr);

                    }

                    else

                        return col1;

                }



                ENDCG

            }

        }

            FallBack "Diffuse"

}

 

Any help greatly appreciated

 

any screenshots?

colors from webcam will be noisy/not exact colors, so i would expect it to be grainy..

i tried something similar with hsv matching, but that was grainy in webcam input also:
https://unitycoder.com/blog/2018/10/19/hsv-debugging-shader/

 

Thanks for the reply... Wondering how to proceed on this though or maybe can you suggest an alternative if at all possible.

 

Can I take a hue or Value range to reduce noise...

 

i tried that, but still the raw selection wont be smooth/accurate (especially when brightness changes),
i guess could also blur the selection, but not sure how would that look (and it would bleed outside the area).

even in photoshop, after effects, i havent managed get good selection (by color or hue) to use it as a mask.. (if source not evenly lighted, is grainy already etc).

not sure what good options there are,
https://stackoverflow.com/questions/19986198/emulate-photoshops-color-range-algorithm
https://stackoverflow.com/questions/1678457/best-algorithm-for-matching-colours
https://www.researchgate.net/public...image_comparison_based_on_similarity_measures

 

Thanks so much for your help and guidance ... guess I'll need to discern the logic of how to recognise hue gradients...

 

can you help me in one thing though... In my csharp code above I am trying to pick colour using raycast and it seems to pick the wrong colour. Can you point me in the direction where I might be going wrong ...thanks.

 

see this example with texturecoord from raycast hit, that should work (to get pixel coordinate)
https://docs.unity3d.com/ScriptReference/RaycastHit-textureCoord.html

this would read pixel from the screen, then no need raycasts and all that (can make it 1 pixel size)
https://docs.unity3d.com/ScriptReference/Texture2D.ReadPixels.html

 

Thanks so very much .... for such swift response ... It is really helpful