How can I convert from YV21 to NV21 on the VideoBackground shader on a unity app with vuforia?

My team is developing an augmented reality app for android and ios devices on Unity 3D 2018.1.3f1 and vuforia cloud recognition 7.1.31, the problem is that in some devices the video format is different, for example on the ZTE v10 we got this graphic bug and on a Moto G5 plus it works fine

We are using this shader for the video background

[code=CSharp]Shader "Custom/VideoBackground" {

    // Used to render the Vuforia Video Background



    Properties

    {

        [NoScaleOffset] _MainTex("Texture", 2D) = "white" {}

        [NoScaleOffset] _UVTex("UV Texture", 2D) = "white" {}

    }

        SubShader

    {

        Tags {"Queue" = "geometry-11" "RenderType" = "opaque" }

        Pass {

            ZWrite Off

            Cull Off

            Lighting Off



            CGPROGRAM



            #pragma multi_compile VUFORIA_RGB VUFORIA_YUVNV12 VUFORIA_YUVNV21



            #pragma vertex vert

            #pragma fragment frag



            #include "UnityCG.cginc"





            sampler2D _MainTex;

            float4 _MainTex_ST;

#if (VUFORIA_YUVNV12 || VUFORIA_YUVNV21)

            sampler2D _UVTex;

            float4 _UVTex_ST;

#endif



            struct v2f {

                float4  pos : SV_POSITION;

                float2  uv : TEXCOORD0;

#if (VUFORIA_YUVNV12 || VUFORIA_YUVNV21)

                float2  uv2 : TEXCOORD1;

#endif

            };



            v2f vert(appdata_base v)

            {

                v2f o;

                o.pos = UnityObjectToClipPos(v.vertex);

                o.uv = TRANSFORM_TEX(v.texcoord, _MainTex);

#if (VUFORIA_YUVNV12 || VUFORIA_YUVNV21)

                o.uv2 = TRANSFORM_TEX(v.texcoord, _UVTex);

#endif                              

                return o;

            }



#if (VUFORIA_YUVNV12 || VUFORIA_YUVNV21)

               half4 frag(v2f i) : COLOR

            {

                half4 c;

                half2 uv = tex2D(_UVTex, i.uv2).rg;

                float y = tex2D(_MainTex, i.uv).r;



#if VUFORIA_YUVNV12               

                half4 v4yuv1 = half4(y, uv, 1.0);



                c.r = dot(half4(1.1640625,  0.000000000,  1.5957031250, -0.87060546875), v4yuv1);

                c.g = dot(half4(1.1640625, -0.390625000, -0.8134765625,  0.52929687500), v4yuv1);

                c.b = dot(half4(1.1640625,  2.017578125,  0.0000000000, -1.08154296875), v4yuv1);

                c.a = 1.0;

#else              

                half4 v4yuv1 = half4(y, uv, 1.0);



                c.r = dot(half4(1.1640625,  1.5957031250,  0.000000000, -0.87060546875), v4yuv1);

                c.g = dot(half4(1.1640625, -0.8134765625, -0.390625000,  0.52929687500), v4yuv1);

                c.b = dot(half4(1.1640625,  0.0000000000,  2.017578125, -1.08154296875), v4yuv1);

                c.a = 1.0;

#endif



#ifdef UNITY_COLORSPACE_GAMMA

                return c;

#else

                return fixed4(GammaToLinearSpace(c.rgb), c.a);

#endif   

            }

#else

            half4 frag(v2f i) : COLOR

            {

                half4 c = tex2D(_MainTex, i.uv);



                c.rgb = c.rgb;

                c.a = 1.0;



#ifdef UNITY_COLORSPACE_GAMMA

                return c;

#else

                return fixed4(GammaToLinearSpace(c.rgb), c.a);

#endif   

            }

#endif

            ENDCG

        }

    }

        Fallback "Legacy Shaders/Diffuse"

}

[/code]

Other problem is that with only that shader we got a black screen, we use this code to change the image pixel format of the camera but then we got the graphic bug from the image

Code (CSharp):

1. #region PRIVATE_MEMBERS
2.  
3. private Image.PIXEL_FORMAT mPixelFormat = Image.PIXEL_FORMAT.UNKNOWN_FORMAT;
4.  
5. private bool mAccessCameraImage = true;
6. private bool mFormatRegistered = false;
7.  
8. #endregion // PRIVATE_MEMBERS
9.  
10. #region MONOBEHAVIOUR_METHODS
11.  
12. void Start()
13. {
14.  
15.     //#if UNITY_EDITOR
16.     //        mPixelFormat = Image.PIXEL_FORMAT.GRAYSCALE; // Need Grayscale for Editor
17.     //#else
18.     //        //mPixelFormat = Image.PIXEL_FORMAT.; // Use RGB888 for mobile
19.     //#endif
20.  
21.     mPixelFormat = Image.PIXEL_FORMAT.UNKNOWN_FORMAT; // Use RGB888 for mobile
22.  
23.     // Register Vuforia life-cycle callbacks:
24.     VuforiaARController.Instance.RegisterVuforiaStartedCallback(OnVuforiaStarted);
25.     VuforiaARController.Instance.RegisterTrackablesUpdatedCallback(OnTrackablesUpdated);
26.     VuforiaARController.Instance.RegisterOnPauseCallback(OnPause);
27.  
28. }
29.  
30. void OnApplicationPaused(bool paused)
31. {
32.     if (paused)
33.     {
34.         // invalidate registered format if app has been paused
35.         mFormatRegistered = false;
36.     }
37. }
38. void OnTrackablesUpdated()
39. {
40.     if (mFormatRegistered)
41.     {
42.         if (mAccessCameraImage)
43.         {
44.             Vuforia.Image image = CameraDevice.Instance.GetCameraImage(mPixelFormat);
45.  
46.             if (image != null)
47.             {
48.                 Debug.Log(
49.                     "\nImage Format: " + image.PixelFormat +
50.                     "\nImage Size:   " + image.Width + "x" + image.Height +
51.                     "\nBuffer Size:  " + image.BufferWidth + "x" + image.BufferHeight +
52.                     "\nImage Stride: " + image.Stride + "\n"
53.                 );
54.  
55.                 byte[] pixels = image.Pixels;
56.                 byte[] output = new byte[pixels.Length];
57.  
58.                 pixels.CopyTo(output, 0);
59.  
60.                 if (image.Pixels != null && image.Pixels.Length > 0)
61.                 {
62.                     Debug.Log(image.Pixels.Length);
63.                     Debug.Log(
64.                         "\nImage pixels: " +
65.                         image.Pixels[0] + ", " +
66.                         image.Pixels[1] + ", " +
67.                         image.Pixels[2] + ", ...\n"
68.                     );
69.                 }
70.             }
71.  
72.         }
73.     }
74.     else
75.     {
76.         RegisterFormat();
77.     }
78. }
79.     void RegisterFormat()
80. {
81.     if (CameraDevice.Instance.SetFrameFormat(mPixelFormat, true))
82.     {
83.         Debug.Log("Successfully registered camera pixel format " + mPixelFormat.ToString());
84.         mFormatRegistered = true;
85.     }
86.     else
87.     {
88.         Debug.LogError("Failed to register camera pixel format " + mPixelFormat.ToString());
89.         mFormatRegistered = false;
90.     }
91. }

There is anyway we can convert the video format from YV21 to NV21 on the shader?

Thanks in advance.

 

I have the same issue on my phone. Tried with unity 2019.1 and 2018.3.

 

Same issue here. The glitch happens on two of the three phones I have tested. Has anybody found a solution to this?

 

I just cloned out the Vufo shader folder and fixed linking and whatever errored to fix it figuring that the shader they use for the background plane is gonna be THE only shader mix that works and straying from it DID cause issues

That said Command Buffers dont seem to play nice with the ipad and vufo setup