Question Oculus quest screen sharing with Agora

I am using Agora.io in Unity to perform screen sharing and it works well when there are two desktop PCs involved. Now I am trying to achieve the same with Oculus Quest and one PC. The PC will have a raw image texture that shows Oculus screen view. Unfortunately there is no input at all, there is only a black screen. But remind you it works well when two PCs or even an android phone is connected, it displays the screen view. It only does not work when Oculus Quest is connected. I have even given all permissions required for Oculus to achieve this but still makes no sense why it won't work.

Code (CSharp):

1. using System;
2. using System.Collections;
3. using System.Collections.Generic;
4. using System.Globalization;
5. using System.Runtime.InteropServices;
6. using agora_gaming_rtc;
7. using UnityEngine;
8. using UnityEngine.UI;
9. public class ScreenShare : MonoBehaviour {
10.     Texture2D mTexture;
11.     Rect mRect;
12.     [SerializeField]
13.     private string appId = "Your_AppID";
14.     [SerializeField]
15.     private string channelName = "agora";
16.     public IRtcEngine mRtcEngine;
17.     int i = 100;
18.     void Start () {
19.         Debug.Log ("ScreenShare Activated");
20.         mRtcEngine = IRtcEngine.getEngine (appId);
21.         // enable log
22.         mRtcEngine.SetLogFilter (LOG_FILTER.DEBUG | LOG_FILTER.INFO | LOG_FILTER.WARNING | LOG_FILTER.ERROR | LOG_FILTER.CRITICAL);
23.         // set callbacks (optional)
24.         mRtcEngine.SetParameters ("{\"rtc.log_filter\": 65535}");
25.         //Configure the external video source
26.         mRtcEngine.SetExternalVideoSource (true, false);
27.         // Start video mode
28.         mRtcEngine.EnableVideo ();
29.         // allow camera output callback
30.         mRtcEngine.EnableVideoObserver ();
31.         // join channel
32.         mRtcEngine.JoinChannel (channelName, null, 0);
33.         //Create a rectangle width and height of the screen
34.         mRect = new Rect (0, 0, Screen.width, Screen.height);
35.         //Create a texture the size of the rectangle you just created
36.         mTexture = new Texture2D ((int) mRect.width, (int) mRect.height, TextureFormat.BGRA32, false);
37.     }
38.     void Update () {
39.         //Start the screenshare Coroutine
40.         StartCoroutine (shareScreen ());
41.     }
42.     //Screen Share
43.     IEnumerator shareScreen () {
44.         yield return new WaitForEndOfFrame ();
45.         //Read the Pixels inside the Rectangle
46.         mTexture.ReadPixels (mRect, 0, 0);
47.         //Apply the Pixels read from the rectangle to the texture
48.         mTexture.Apply ();
49.         // Get the Raw Texture data from the the from the texture and apply it to an array of bytes
50.         byte[] bytes = mTexture.GetRawTextureData ();
51.         // Make enough space for the bytes array
52.         int size = Marshal.SizeOf (bytes[0]) * bytes.Length;
53.         // Check to see if there is an engine instance already created
54.         IRtcEngine rtc = IRtcEngine.QueryEngine ();
55.         //if the engine is present
56.         if (rtc != null) {
57.             //Create a new external video frame
58.             ExternalVideoFrame externalVideoFrame = new ExternalVideoFrame ();
59.             //Set the buffer type of the video frame
60.             externalVideoFrame.type = ExternalVideoFrame.VIDEO_BUFFER_TYPE.VIDEO_BUFFER_RAW_DATA;
61.             // Set the video pixel format
62.             externalVideoFrame.format = ExternalVideoFrame.VIDEO_PIXEL_FORMAT.VIDEO_PIXEL_BGRA;
63.             //apply raw data you are pulling from the rectangle you created earlier to the video frame
64.             externalVideoFrame.buffer = bytes;
65.             //Set the width of the video frame (in pixels)
66.             externalVideoFrame.stride = (int) mRect.width;
67.             //Set the height of the video frame
68.             externalVideoFrame.height = (int) mRect.height;
69.             //Remove pixels from the sides of the frame
70.             externalVideoFrame.cropLeft = 10;
71.             externalVideoFrame.cropTop = 10;
72.             externalVideoFrame.cropRight = 10;
73.             externalVideoFrame.cropBottom = 10;
74.             //Rotate the video frame (0, 90, 180, or 270)
75.             externalVideoFrame.rotation = 180;
76.             // increment i with the video timestamp
77.             externalVideoFrame.timestamp = i++;
78.             //Push the external video frame with the frame we just created
79.             int a = rtc.PushVideoFrame (externalVideoFrame);
80.             Debug.Log (" pushVideoFrame =       " + a);
81.         }
82.     }
83. }

 

Did you ever find a solution to this? Im having the same issues

 

+1

 

I am also facing the same issue.

 

Switching to OpenGLES3 from Vulkan solved the problem.

I guess ReadPixels() function is not working in Quest 2 with Vulkan API

 

This seems to me like a custom shader if I am not wrong. This is tricky in VR as certain things needs to be done in the correct way. Try reading up in the Documentation about creating and coding shaders for VR Singlepass