Search Unity

  1. Megacity Metro Demo now available. Download now.
    Dismiss Notice
  2. Unity support for visionOS is now available. Learn more in our blog post.
    Dismiss Notice

Bug CammandBuffer.IssuePluginEvent eyeIndex param inconsistent with Low Level Rendering Plugin

Discussion in 'VR' started by austintsmith13, Jan 7, 2022.

  1. austintsmith13

    austintsmith13

    Joined:
    Dec 27, 2018
    Posts:
    8
    Hello, I was exploring to build an application that compiles and renders Shaders at runtime and I think I came across a Unity Bug.

    Unity Version: 2019.4.16 and 2021.1.19

    Bug Description:
    Link to a giff of the issue. https://gyazo.com/6df31febec8bf18fe9bdc4e042f10971
    It seems like the eye index is incorrect for one frame every few seconds. This gets set from UnityRenderingExtEvent, my guess is its some sort of multithreading race condition. I've tried sending the eye index from both C# onPreRendered and from the C++ Extended interface.

    EDIT: I did a little bit more tinkering and found out I cant get this to render to the left eye without flickering every few seconds. So this has nothing to do with the eye index being sent incorrectly.
    My guess is the right eye render texture gets copied over to the left on some condition that is met every few seconds.

    How to reproduce:
    Create a native render plugin, set up project as VR with Multi-Pass. Setup CommandBuffer to call "IssuePluginEventAndData". Marshal data on PreRender. One of the eyes is always flickering.

    Some code that exhibits the isssue below.
    Code (CSharp):
    1. public class CustomRenderPluginExample : MonoBehaviour
    2. {
    3.     public Light m_Light;
    4.     public Camera m_Camera = null;
    5.     public MeshFilter m_MeshFilter = null;
    6.     public Texture2D m_MeshTexture = null;
    7.     public Transform m_WorldObjectTransform = null;
    8.     public Material m_Mat;
    9.     // SetupResources and GetRenderEventFunc are the two methods found in our native rendering plugin.
    10.     [DllImport("NativeRenderingPlugin")]
    11.     private static extern void SetupResources(System.IntPtr nativeIndexBuffer, System.IntPtr nativeVertexBuffer, System.IntPtr nativeTextureResource);
    12.  
    13.     // GetRenderEventFunc will return a native function pointer to our native DoRenderEvent method
    14.     // which will ultimately intialize our native graphics state object and handle the native rendering of
    15.     // our mesh.
    16.     [DllImport("NativeRenderingPlugin")]
    17.     private static extern System.IntPtr GetRenderEventFunc();
    18.     CommandBuffer cb = null;
    19.     CommandBuffer leftCb = null;
    20.     CommandBuffer rightCb = null;
    21.     CommandBuffer shadowCb = null;
    22.     Matrix4x4 rightProjectionMatrix;
    23.     Matrix4x4 leftProjectionMatrix;
    24.     System.IntPtr nativeRenderingDataPtr;
    25.  
    26.     enum CustomRenderEvent
    27.     {
    28.         // 3245 is a random number I made up.
    29.         // I figured it could be useful to send an event id
    30.         // to the native plugin which corresponds to when
    31.         // the mesh is being rendered in the render pipeline.
    32.         AfterForwardOpaque = 3245,
    33.     }
    34.  
    35.     // These are the matrices we are going to send from Unity to
    36.     // the native rendering plugin.
    37.     [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)]
    38.     struct NativeRenderingData
    39.     {
    40.         public const int Size =
    41.             sizeof(float) * 16 +
    42.             sizeof(float) * 16 +
    43.             sizeof(float) * 16 +
    44.             sizeof(float) * 16;
    45.  
    46.         [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)]
    47.         public float[] localToWorldMatrix;
    48.         [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)]
    49.         public float[] worldToViewMatrix;
    50.         [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)]
    51.         public float[] leftViewToProjectionMatrix;
    52.  
    53.         [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)]
    54.         public float[] rightViewToProjectionMatrix;
    55.     };
    56.  
    57.     NativeRenderingData nativeRenderingData = new NativeRenderingData();
    58.     Matrix4x4[] shadowTestMatrix = new Matrix4x4[1];
    59.  
    60.     // Direct referenced to a separate instance of mesh that way copied over, unity will move S*** around even if its referenced through the lifetime of the game.
    61.     Mesh mesh;
    62.  
    63.     // Use this for initialization
    64.     void Awake()
    65.     {
    66.         GameObject meshObject = (GameObject)Resources.Load<GameObject>("tekka");
    67.         Mesh meshC = meshObject.GetComponentInChildren<MeshFilter>().sharedMesh;
    68.         mesh = new Mesh();
    69.         mesh.vertices = meshC.vertices;
    70.         mesh.triangles = meshC.triangles;
    71.         mesh.uv = meshC.uv;
    72.         mesh.normals = meshC.normals;
    73.         mesh.RecalculateBounds();
    74.         mesh.RecalculateNormals();
    75.         mesh.RecalculateTangents();
    76.  
    77.         System.IntPtr nativeIndexBufferPtr = mesh.GetNativeIndexBufferPtr();
    78.         System.IntPtr nativeVertexBufferPtr = mesh.GetNativeVertexBufferPtr(0);
    79.         System.IntPtr nativeTexturePtr = m_MeshTexture.GetNativeTexturePtr();
    80.  
    81.         // Pass the native graphics API index/vertex/texture
    82.         // pointers to the native rendering plugin.
    83.         SetupResources(nativeIndexBufferPtr, nativeVertexBufferPtr, nativeTexturePtr);
    84.  
    85.         nativeRenderingData = new NativeRenderingData();
    86.         nativeRenderingData.localToWorldMatrix = new float[16];
    87.         nativeRenderingData.worldToViewMatrix = new float[16];
    88.         nativeRenderingData.rightViewToProjectionMatrix = new float[16];
    89.         nativeRenderingData.leftViewToProjectionMatrix = new float[16];
    90.  
    91.         // Allocate unmanaged memory to hold all three of our matrices that we
    92.         // weish to send to our native rendering plugin.
    93.         nativeRenderingDataPtr = Marshal.AllocHGlobal(NativeRenderingData.Size);
    94.  
    95.         cb = new CommandBuffer();
    96.         leftCb = new CommandBuffer();
    97.         rightCb = new CommandBuffer();
    98.         shadowCb = new CommandBuffer();
    99.  
    100.         shadowTestMatrix[0] = m_WorldObjectTransform.localToWorldMatrix;
    101.         shadowCb.DrawMeshInstanced(mesh, 0, m_Mat, 2, shadowTestMatrix, 1, null);
    102.  
    103.         leftCb.IssuePluginEventAndData(GetRenderEventFunc(), (int)CustomRenderEvent.AfterForwardOpaque, nativeRenderingDataPtr);
    104.  
    105.         m_Camera.AddCommandBuffer(CameraEvent.AfterForwardOpaque, leftCb);
    106.  
    107.         m_Light.AddCommandBuffer(LightEvent.AfterShadowMapPass, shadowCb);
    108.     }
    109.  
    110.  
    111.     // There might be a better way to do this but
    112.     // I want to marshal the C# Matrix4x4 data to C++
    113.     // without any marshaling issues.
    114.     void MatrixToFloatArray(Matrix4x4 m, ref float[] outputFloatArray)
    115.     {
    116.         outputFloatArray[0] = m.m00;
    117.         outputFloatArray[1] = m.m01;
    118.         outputFloatArray[2] = m.m02;
    119.         outputFloatArray[3] = m.m03;
    120.  
    121.         outputFloatArray[4] = m.m10;
    122.         outputFloatArray[5] = m.m11;
    123.         outputFloatArray[6] = m.m12;
    124.         outputFloatArray[7] = m.m13;
    125.  
    126.         outputFloatArray[8] = m.m20;
    127.         outputFloatArray[9] = m.m21;
    128.         outputFloatArray[10] = m.m22;
    129.         outputFloatArray[11] = m.m23;
    130.  
    131.         outputFloatArray[12] = m.m30;
    132.         outputFloatArray[13] = m.m31;
    133.         outputFloatArray[14] = m.m32;
    134.         outputFloatArray[15] = m.m33;
    135.     }
    136.  
    137.     private void UpdateNativeData()
    138.     {
    139.  
    140.         Matrix4x4 rightProjectionMatrix = m_Camera.GetStereoProjectionMatrix(Camera.StereoscopicEye.Right);
    141.         Matrix4x4 leftProjectionMatrix = m_Camera.GetStereoProjectionMatrix(Camera.StereoscopicEye.Left);
    142.  
    143.         leftProjectionMatrix = GL.GetGPUProjectionMatrix(leftProjectionMatrix, true);
    144.         rightProjectionMatrix = GL.GetGPUProjectionMatrix(rightProjectionMatrix, true);
    145.  
    146.         MatrixToFloatArray(m_WorldObjectTransform.localToWorldMatrix, ref nativeRenderingData.localToWorldMatrix);
    147.         MatrixToFloatArray(m_Camera.worldToCameraMatrix, ref nativeRenderingData.worldToViewMatrix);
    148.         MatrixToFloatArray(leftProjectionMatrix, ref nativeRenderingData.leftViewToProjectionMatrix);
    149.         MatrixToFloatArray(rightProjectionMatrix, ref nativeRenderingData.rightViewToProjectionMatrix);
    150.  
    151.         Marshal.StructureToPtr(nativeRenderingData, nativeRenderingDataPtr, true);
    152.     }
    153.  
    154.     public bool test = true;
    155.  
    156.     private void UpdateNativeDataHack()
    157.     {
    158.         var m = m_Camera.projectionMatrix;
    159.         var activeEye = m_Camera.stereoActiveEye;
    160.         if (activeEye == Camera.MonoOrStereoscopicEye.Left && test)
    161.         {
    162.             leftProjectionMatrix = m_Camera.GetStereoProjectionMatrix(Camera.StereoscopicEye.Left);
    163.         }
    164.  
    165.         if (activeEye == Camera.MonoOrStereoscopicEye.Right && test)
    166.         {
    167.             rightProjectionMatrix = m_Camera.GetStereoProjectionMatrix(Camera.StereoscopicEye.Right);
    168.         }
    169.  
    170.  
    171.         leftProjectionMatrix = GL.GetGPUProjectionMatrix(leftProjectionMatrix, true);
    172.         rightProjectionMatrix = GL.GetGPUProjectionMatrix(rightProjectionMatrix, true);
    173.  
    174.         MatrixToFloatArray(m_WorldObjectTransform.localToWorldMatrix, ref nativeRenderingData.localToWorldMatrix);
    175.         MatrixToFloatArray(m_Camera.worldToCameraMatrix, ref nativeRenderingData.worldToViewMatrix);
    176.         MatrixToFloatArray(leftProjectionMatrix, ref nativeRenderingData.leftViewToProjectionMatrix);
    177.         MatrixToFloatArray(rightProjectionMatrix, ref nativeRenderingData.rightViewToProjectionMatrix);
    178.  
    179.         // Copy our managed nativeRenderingData into the unmanaged memory pointed to by
    180.         // the nativeRenderingDataPtr pointer.
    181.         Marshal.StructureToPtr(nativeRenderingData, nativeRenderingDataPtr, true);
    182.     }
    183.  
    184.     private void OnPreRender()
    185.     {
    186.         UpdateNativeDataHack();
    187.     }
    188.  
    189.     private void OnDestroy()
    190.     {
    191.         // Be a good Citizen and clean up after yourself.  =D
    192.         cb.Release();
    193.         Marshal.FreeHGlobal(nativeRenderingDataPtr);
    194.         nativeRenderingDataPtr = System.IntPtr.Zero;
    195.     }
    196. }
    197.  
    C++ code
    Code (CSharp):
    1. #include "Microsoft/DirectX/include/D3D11.h"
    2. #include "Unity/IUnityInterface.h"
    3. #include "Unity/IUnityGraphics.h"
    4. #include "Unity/IUnityGraphicsD3D11.h"
    5. #include "Unity/IUnityRenderingExtensions.h"
    6. #include "UnityRenderLogic.h"
    7.  
    8. using namespace std;
    9.  
    10. static IUnityInterfaces* s_UnityInterfaces = NULL;
    11. static IUnityGraphics* s_Graphics = NULL;
    12. static UnityGfxRenderer s_RendererType = kUnityGfxRendererNull;
    13. int eyeIndex = 0;
    14.  
    15. static void UNITY_INTERFACE_API OnGraphicsDeviceEvent(UnityGfxDeviceEventType eventType);
    16. static void DoEventGraphicsDeviceD3D11(UnityGfxDeviceEventType eventType);
    17.  
    18. static bool initialized = false;
    19.  
    20. extern "C"
    21. {
    22.     // Unity plugin load event
    23.     void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API UnityPluginLoad(IUnityInterfaces* unityInterfaces)
    24.     {
    25.         s_UnityInterfaces = unityInterfaces;
    26.         s_Graphics = unityInterfaces->Get<IUnityGraphics>();
    27.  
    28.         s_Graphics->RegisterDeviceEventCallback(OnGraphicsDeviceEvent);
    29.  
    30.         // Run OnGraphicsDeviceEvent(initialize) manually on plugin load
    31.         // to not miss the event in case the graphics device is already initialized
    32.         OnGraphicsDeviceEvent(kUnityGfxDeviceEventInitialize);
    33.     }
    34.  
    35.     // Unity plugin unload event
    36.     void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API UnityPluginUnload()
    37.     {
    38.         s_Graphics->UnregisterDeviceEventCallback(OnGraphicsDeviceEvent);
    39.     }
    40.  
    41.     void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API DoRenderEvent(int eventId, int* data)
    42.     {
    43.         if (!initialized)
    44.         {
    45.             s_UnityRenderLogic.Initialize(s_UnityInterfaces, eventId, data);
    46.             initialized = true;
    47.         }
    48.  
    49.         s_UnityRenderLogic.Render(s_UnityInterfaces, eventId, data);
    50.     }
    51.  
    52.     UnityRenderingEvent UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API GetRenderEventFunc()
    53.     {
    54.         return DoRenderEvent;
    55.     }
    56.  
    57.     void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API SetupResources(void* indexBuffer, void* vertexBuffer, void* texture)
    58.     {
    59.         s_UnityRenderLogic.SetupResources(indexBuffer, vertexBuffer, texture);
    60.     }
    61.  
    62.     void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API UnityRenderingExtEvent(UnityRenderingExtEventType event, void* data)
    63.     {
    64.         switch (event)
    65.         {
    66.         case UnityRenderingExtEventType::kUnityRenderingExtEventSetStereoEye:
    67.  
    68.             // Seems to not update eyeIndex in time. Maybe Render thread gets a little behind?
    69.             eyeIndex = reinterpret_cast<int>(data);
    70.             break;
    71.         case UnityRenderingExtEventType::kUnityRenderingExtEventSetStereoTarget:
    72.             //eyeIndex = reinterpret_cast<int>(data);
    73.             break;
    74.         case UnityRenderingExtEventType::kUnityRenderingExtEventBeforeDrawCall:
    75.             //auto params = reinterpret_cast<UnityRenderingExtBeforeDrawCallParams*>(data);
    76.             //eyeIndex = params->eyeIndex;
    77.        
    78.             break;
    79.         }
    80.     }
    81. }
    82.  
    83. static void UNITY_INTERFACE_API OnGraphicsDeviceEvent(UnityGfxDeviceEventType eventType)
    84. {
    85.     UnityGfxRenderer currentDeviceType = s_RendererType;
    86.  
    87.     switch (eventType)
    88.     {
    89.         case kUnityGfxDeviceEventInitialize:
    90.         {
    91.             //TODO: user initialization code
    92.             initialized = false;
    93.             break;
    94.         }
    95.         case kUnityGfxDeviceEventShutdown:
    96.         {
    97.             //TODO: user shutdown code
    98.             s_UnityRenderLogic.Shutdown(s_UnityInterfaces);
    99.             initialized = false;
    100.             break;
    101.         }
    102.         case kUnityGfxDeviceEventBeforeReset:
    103.         {
    104.             //TODO: user Direct3D 9 code
    105.             break;
    106.         }
    107.         case kUnityGfxDeviceEventAfterReset:
    108.         {
    109.             //TODO: user Direct3D 9 code
    110.             break;
    111.         }
    112.     };
    113.  
    114.     if (currentDeviceType == kUnityGfxRendererD3D11)
    115.     {
    116.         DoEventGraphicsDeviceD3D11(eventType);
    117.     }
    118. }
    119.  
    120. static void DoEventGraphicsDeviceD3D11(UnityGfxDeviceEventType eventType)
    121. {
    122.     if (eventType == kUnityGfxDeviceEventInitialize)
    123.     {
    124.  
    125.     }
    126.     else if (eventType == kUnityGfxDeviceEventShutdown)
    127.     {
    128.    
    129.     }
    130. }
    131.  
     
    Last edited: Jan 8, 2022