Question how to speed up code that computes render-to-render optical flow and saves as separate PNG files

I'm running the C# code below, which computes optical flow maps and saves them as PNGs during gameplay, using Unity with a VR headset that forces an upper limit of 90 FPS. Project is on HDRP. Without this code, the project runs smoothly at 90 FPS. To run this code on top of the same project consistently above 80 FPS, I had to use WaitForSeconds(0.2f) in the coroutine but the ideal scenario would be to compute and save the optical flow map for every frame of the game, or at least with a lower delay about 0.01 seconds. I'm already using AsyncGPUReadback and WriteAsync.

• Main Question: How can I speed up this code further?
• Side Question: Any way I can dump the calculated optical flow maps as consecutive rows in a CSV file so that it would write on a single file rather than creating a separate PNG for each map? Or would this be even slower?

Code (CSharp):

1.  
2. using System.Collections;
3. using UnityEngine;
4. using System.IO;
5. using UnityEngine.Rendering;
6.  
7. namespace OpticalFlowAlternative
8. {
9.  
10.     public class OpticalFlow : MonoBehaviour {
11.  
12.         protected enum Pass {
13.             Flow = 0,
14.             DownSample = 1,
15.             BlurH = 2,
16.             BlurV = 3,
17.             Visualize = 4
18.         };
19.  
20.         public RenderTexture Flow { get { return resultBuffer; } }
21.  
22.         [SerializeField] protected Material flowMaterial;
23.         protected RenderTexture prevFrame, flowBuffer, resultBuffer, renderTexture, rt;
24.  
25.         public string customOutputFolderPath = "";
26.         private string filepathforflow;
27.         private int imageCount = 0;
28.  
29.         int targetTextureWidth, targetTextureHeight;
30.  
31.         private EyeTrackingV2 eyeTracking;
32.  
33.         protected void Start () {
34.             eyeTracking = GameObject.Find("XR Rig").GetComponent<EyeTrackingV2>();
35.             targetTextureWidth = Screen.width / 16;
36.             targetTextureHeight = Screen.height / 16;
37.             flowMaterial.SetFloat("_Ratio", 1f * Screen.height / Screen.width);
38.  
39.             renderTexture = new RenderTexture(targetTextureWidth, targetTextureHeight, 0);
40.             rt = new RenderTexture(Screen.width, Screen.height, 0);
41.  
42.             StartCoroutine("StartCapture");
43.         }
44.  
45.         protected void LateUpdate()
46.         {
47.             eyeTracking.flowCount = imageCount;
48.         }
49.  
50.         protected void OnDestroy ()
51.         {
52.             if(prevFrame != null)
53.             {
54.                 prevFrame.Release();
55.                 prevFrame = null;
56.  
57.                 flowBuffer.Release();
58.                 flowBuffer = null;
59.  
60.                 rt.Release();
61.                 rt = null;
62.  
63.                 renderTexture.Release();
64.                 renderTexture = null;
65.             }
66.         }
67.  
68.        IEnumerator StartCapture()
69.         {
70.             while (true)
71.             {
72.                 yield return new WaitForSeconds(0.2f);
73.                
74.                 ScreenCapture.CaptureScreenshotIntoRenderTexture(rt);
75.                 //compensating for image flip
76.                 Graphics.Blit(rt, renderTexture, new Vector2(1, -1), new Vector2(0, 1));
77.  
78.                 if (prevFrame == null)
79.                 {
80.                     Setup(targetTextureWidth, targetTextureHeight);
81.                     Graphics.Blit(renderTexture, prevFrame);
82.                 }
83.  
84.                 flowMaterial.SetTexture("_PrevTex", prevFrame);
85.  
86.                 //calculating motion flow frame here
87.                 Graphics.Blit(renderTexture, flowBuffer, flowMaterial, (int)Pass.Flow);
88.                 Graphics.Blit(renderTexture, prevFrame);
89.                
90.                 AsyncGPUReadback.Request(flowBuffer, 0, TextureFormat.ARGB32, OnCompleteReadback);
91.             }
92.         }
93.  
94.         void OnCompleteReadback(AsyncGPUReadbackRequest request)
95.         {
96.             if (request.hasError)
97.                 return;
98.  
99.             var tex = new Texture2D(targetTextureWidth, targetTextureHeight, TextureFormat.ARGB32, false);
100.             tex.LoadRawTextureData(request.GetData<uint>());
101.             tex.Apply();
102.  
103.             WriteTextureAsync(tex);
104.         }
105.  
106.         async void WriteTextureAsync(Texture2D tex)
107.         {
108.  
109.             imageCount++;
110.            
111.             filepathforflow = customOutputFolderPath + imageCount + ".png";
112.             var stream = new FileStream(filepathforflow, FileMode.OpenOrCreate);
113.             var bytes = tex.EncodeToPNG();
114.             await stream.WriteAsync(bytes, 0, bytes.Length);
115.         }
116.  
117.         protected void Setup(int width, int height)
118.         {
119.             prevFrame = new RenderTexture(width, height, 0);
120.             prevFrame.format = RenderTextureFormat.ARGBFloat;
121.             prevFrame.wrapMode = TextureWrapMode.Repeat;
122.             prevFrame.Create();
123.  
124.             flowBuffer = new RenderTexture(width, height, 0);
125.             flowBuffer.format = RenderTextureFormat.ARGBFloat;
126.             flowBuffer.wrapMode = TextureWrapMode.Repeat;
127.             flowBuffer.Create();
128.         }
129.     }
130.  
131. }

 

do you need to display those textures, or just save into file?
maybe then can skip texture creation and applying. (that is usually slow)

writing to disk could be done in separate thread (file.writeallbytes), if just dump raw bytearray there.

and check deep profiler, which part is slowest currently.

 

no I don't need to display them, just need to save the flowbuffer texture, exactly which lines should I discard then?

can you elaborate on this? or an example code would be great. how do I do this in a separate thread? apparently there is also a File.WriteAllBytesAsync, should I prefer this?

 

i'd then remove these:

Code (CSharp):

1. var tex = new Texture2D(targetTextureWidth, targetTextureHeight, TextureFormat.ARGB32, false);
2. tex.LoadRawTextureData(request.GetData<uint>());
3. tex.Apply();
4.  

and just dump "request.GetData<uint>()" into file. (can process that raw data into texture later when needed)
either try that WriteAllBytesAsync, or push into Queue and have another thread saving it out.

or if csv data is fine too:
- add request.GetData<uint>() into Queue
- have another thread processing the Queue data into csv file or one csv file per frame (that you can then merge later)

 

ok I'm trying to save the render texture directly as raw bytes. However when I read from the saved file and re-save it as PNG, to make sure I did the right thing, I see that PNG file is just blank.
What am I doing wrong?

This is the code I'm using to save render texture as raw bytes:

Code (CSharp):

1. IEnumerator StartCapture()
2.         {
3.             while (true)
4.             {
5.                 yield return new WaitForSeconds(0.2f);
6.                 ScreenCapture.CaptureScreenshotIntoRenderTexture(rt);
7.                 //compensating for image flip
8.                 Graphics.Blit(rt, renderTexture, new Vector2(1, -1), new Vector2(0, 1));
9.  
10.                if (prevFrame == null)
11.                 {
12.                     Setup(targetTextureWidth, targetTextureHeight);
13.                     Graphics.Blit(renderTexture, prevFrame);
14.                 }
15.  
16.                 flowMaterial.SetTexture("_PrevTex", prevFrame);
17.  
18.                 //calculating motion flow frame here
19.                 Graphics.Blit(renderTexture, flowBuffer, flowMaterial, (int)Pass.Flow);
20.                 Graphics.Blit(renderTexture, prevFrame);
21.        
22.                 AsyncGPUReadback.Request(flowBuffer, 0, TextureFormat.ARGB32, OnCompleteReadbackNew);
23.             }
24.  
25.         }
26.  
27.  void OnCompleteReadbackNew(AsyncGPUReadbackRequest request)
28.         {
29.             if (request.hasError)
30.                 return;
31.  
32.             var bytes = request.GetData<byte>();
33.  
34.             WriteTextureAsyncNew(bytes);
35.         }
36.  
37. async void WriteTextureAsyncNew(NativeArray<byte> bytes)
38.         {
39.             imageCount++;
40.    
41.             filepathforflow = customOutputFolderPath + imageCount + ".txt";
42.             await File.WriteAllBytesAsync(filepathforflow, bytes.ToArray());
43.         }

This is the code I'm using to read the saved file and re-save it as PNG:

Code (CSharp):

1. var newBytes = File.ReadAllBytes(customOutputFolderPath + "5.txt");
2. var target = new Texture2D(targetTextureWidth, targetTextureHeight, TextureFormat.ARGB32, false);
3. target.LoadRawTextureData(newBytes);
4. target.Apply();
5. var lastBytes = target.EncodeToJPG();
6. File.WriteAllBytes(customOutputFolderPath + readCount + ".png", lastBytes

 

check what kind of values the file contains, or is it empty?

hexviewer (drag n drop file): https://hexed.it/

 

also could be blank if your alpha channel is zero. Make sure that isn't the case using an image editor.

Also you are encoding as a JPG, not a PNG. Looks like you have a typo in your code.

 

yes exactly! it was a silly mistake.

so far,
- switched to saving textures as raw bytes
- and saving them as rows into a csv file. of course for this i need to encode them as text using Convert.ToBase64String(bytes.ToArray())

with these i can now generate flow maps at 0.1 seconds apart (down from initial 0.2s) while keeping FPS consistently around 90. But still far away from the ideal 0.01s. Can you guys tell me how to write csv in a separate thread? That can make a real difference in speeding up.
Below is the current writing process:

Code (CSharp):

1. IEnumerator StartCapture()
2.         {
3.             while (true)
4.             {
5.                 yield return new WaitForSeconds(0.1f);
6.                 //yield return new WaitForEndOfFrame();
7.  
8.                 //var rt = RenderTexture.GetTemporary(Screen.width, Screen.height, 0);
9.                 ScreenCapture.CaptureScreenshotIntoRenderTexture(rt);
10.                 //compensating for image flip
11.                 Graphics.Blit(rt, renderTexture, new Vector2(1, -1), new Vector2(0, 1));
12.  
13.                 //RenderTexture.ReleaseTemporary(rt);
14.  
15.                 if (prevFrame == null)
16.                 {
17.                     Setup(targetTextureWidth, targetTextureHeight);
18.                     Graphics.Blit(renderTexture, prevFrame);
19.                 }
20.  
21.                 flowMaterial.SetTexture("_PrevTex", prevFrame);
22.  
23.                 //calculating motion flow frame here
24.                 Graphics.Blit(renderTexture, flowBuffer, flowMaterial, (int)Pass.Flow);
25.                 Graphics.Blit(renderTexture, prevFrame);
26.              
27.                 AsyncGPUReadback.Request(flowBuffer, 0, TextureFormat.ARGB32, OnCompleteReadbackNew);
28.             }
29.         }
30.  
31.         void OnCompleteReadbackNew(AsyncGPUReadbackRequest request)
32.         {
33.             if (request.hasError)
34.                 return;
35.  
36.             var bytes = request.GetData<byte>();
37.  
38.             LogMotionData(bytes);
39.         }
40.      
41.         void LogMotionData(NativeArray<byte> bytes)
42.         {
43.             string[] logData = new string[7];
44.             flowCount++;
45.  
46.             logData[0] = targetTextureWidth.ToString();
47.             logData[1] = targetTextureHeight.ToString();
48.  
49.             //store map bytes as Base64String https://www.c-sharpcorner.com/blogs/convert-an-image-to-base64-string-and-base64-string-to-image
50.             logData[2] = Convert.ToBase64String(bytes.ToArray());
51.  
52.             // flow frame number
53.             logData[3] = flowCount.ToString();
54.  
55.             // capture time
56.             logData[4] = DateTime.Now.ToString();
57.  
58.             //Time since started
59.             logData[5] = Time.time.ToString();
60.  
61.             var format = TextureFormat.ARGB32;
62.             logData[6] = format.ToString();
63.  
64.             Log(logData);
65.         }
66.  
67.         // Write given values in the log file
68.         void Log(string[] values)
69.         {
70.             if (!logging || writer == null)
71.                 return;
72.  
73.             string line = "";
74.             for (int i = 0; i < values.Length; ++i)
75.             {
76.                 values[i] = values[i].Replace("\r", "").Replace("\n", ""); // Remove new lines so they don't break csv
77.                 line += values[i] + (i == (values.Length - 1) ? "" : ";"); // Do not add semicolon to last data string
78.             }
79.             writer.WriteLine(line);
80.         }
81.  
82.         public void StartLogging()
83.         {
84.             //Debug.LogWarning("StartLogging called");
85.             if (logging)
86.             {
87.                 Debug.LogWarning("Logging was on when StartLogging was called. No new log was started.");
88.                 return;
89.             }
90.  
91.             logging = true;
92.  
93.             string logPath;
94.             if (useCustomLogPath)
95.             {
96.                 logPath = "D:\\U\\Logs\\" + sceneLoader.sessionName + "\\";
97.                 //logPath = Application.dataPath + "/Logs/" + customLogPath;
98.             }
99.             else
100.             {
101.                 logPath = Application.dataPath + "/Logs/";
102.             }
103.  
104.  
105.             Directory.CreateDirectory(logPath);
106.  
107.             DateTime now = DateTime.Now;
108.             string fileName = string.Format("{0}-{1:00}-{2:00}-{3:00}-{4:00}", now.Year, now.Month, now.Day, now.Hour, now.Minute);
109.  
110.             string path = logPath + sceneName + "_MotionFlow_" + fileName + ".csv";
111.             //string path = Application.dataPath + "/Logs/" + "sample.csv";
112.             writer = new StreamWriter(path);
113.  
114.             Log(ColumnNames);
115.             Debug.Log("Flow Log file started at: " + path);
116.         }
117.  
118.         void StopLogging()
119.         {
120.             if (!logging)
121.                 return;
122.  
123.             if (writer != null)
124.             {
125.                 writer.Flush();
126.                 writer.Close();
127.                 writer = null;
128.             }
129.             logging = false;
130.             Debug.Log("Flow Logging ended");
131.         }
132.  
133.         void OnApplicationQuit()
134.         {
135.             StopLogging();
136.          
137.         }