Question Outputting Motion Vectors data to image

I can see the Motion Vector data, as color changes and lines using the Rederer Debug window. How would I go about outputting that image (with not lines/vectors) colors to an texture/image ? In other words an optical flow.

I know that it has to do with the camera depth texture and I might need to write a shader. But l can't get any specific information, is that the right solution? I tried to get this shader to work but with no luck:
https://github.com/immersive-limit/...ets/ImageSynthesis/Shaders/OpticalFlow.shader

Am headed in the right direction with this?

 

First of all, motion vectors and optical flow are not the same. Motion vectors describe where certain points move on the screen and can only be generated from 3D data. Optical flow describes where points seem to move based on a 2D analysis of the picture (at least that's how I understand it).
See here:
https://dsp.stackexchange.com/a/34942

If you are fine with the format that Unity's motion vectors are stored (I think they are u/v space vectors), you should be able to just read the motion vector render texture with AsyncGPUReadback and save it with ImageConversion.EncodeToExr(). You need to use EXR because PNG doesn't support floating point values. If you need to do this in real time it might get tricky, though.

I think you can get the motion vector texture by calling Shader.GetGlobalTexture("_CameraMotionVectorsTexture") in the Camera.onPostRender callback, not 100% sure, though.

 

Excellent feedback. Thank you @c0d3_m0nk3y.

 

So I was finally able to get this shader working.

Code (CSharp):

1. Shader"Unlit/Test"
2. {
3.     Properties
4.     {
5.         _MainTex ("Texture", 2D) = "white" {}
6.         [Toggle] _drawArrows("DrawArrows ?", float) = 0
7.  
8.         _MinX ("MinX", Float) = 0
9.         _MaxX ("MaxX", Float) = 0
10.  
11.         _MinY ("MinX", Float) = 0
12.         _MaxY ("MaxX", Float) = 0
13.     }
14.     SubShader
15.     {
16.         Tags { "RenderType"="Opaque" }
17.         LOD 100
18.  
19.         Pass
20.         {
21.             CGPROGRAM
22.             #pragma vertex vert
23.             #pragma fragment frag
24.             // make fog work
25.             #pragma multi_compile_fog
26.  
27.             #include "UnityCG.cginc"
28.  
29.             float _MinX;
30.             float _MaxX;
31.             float _MinY;
32.             float _MaxY;
33.  
34.             struct appdata
35.             {
36.                 float4 vertex : POSITION;
37.                 float2 uv : TEXCOORD0;
38.             };
39.  
40.             struct v2f
41.             {
42.                 float2 uv : TEXCOORD0;
43.                 UNITY_FOG_COORDS(1)
44.                 float4 vertex : SV_POSITION;
45.             };
46.  
47.             sampler2D _MainTex;
48.             float4 _MainTex_ST;
49.  
50.             float _drawArrows;
51.             v2f vert (appdata v)
52.             {
53.                 v2f o;
54.                 o.vertex = UnityObjectToClipPos(v.vertex);
55.                 o.uv = TRANSFORM_TEX(v.uv, _MainTex);
56.                 UNITY_TRANSFER_FOG(o,o.vertex);
57.                 return o;
58.             }
59.  
60.             // Debug Motion Vector from https://github.com/Unity-Technologies/Graphics/blob/master/com.unity.render-pipelines.high-definition/Runtime/Debug/DebugFullScreen.shader#L221
61.  
62.             // Motion vector debug utilities
63.             float DistanceToLine(float2 p, float2 p1, float2 p2)
64.             {
65.                 float2 center = (p1 + p2) * 0.5;
66.                 float len = length(p2 - p1);
67.                 float2 dir = (p2 - p1) / len;
68.                 float2 rel_p = p - center;
69.                 return dot(rel_p, float2(dir.y, -dir.x));
70.             }
71.  
72.             float DistanceToSegment(float2 p, float2 p1, float2 p2)
73.             {
74.                 float2 center = (p1 + p2) * 0.5;
75.                 float len = length(p2 - p1);
76.                 float2 dir = (p2 - p1) / len;
77.                 float2 rel_p = p - center;
78.                 float dist1 = abs(dot(rel_p, float2(dir.y, -dir.x)));
79.                 float dist2 = abs(dot(rel_p, dir)) - 0.5 * len;
80.                 return max(dist1, dist2);
81.             }
82.  
83.             float2 SampleMotionVectors(float2 coords)
84.             {
85.                 float4 col = tex2D(_MainTex, coords);
86.  
87.                 // In case material is set as no motion
88.                 if (col.x > 1)
89.                     return 0;
90.                 else
91.                     return col.xy;
92.             }
93.  
94.             float DrawArrow(float2 texcoord, float body, float head, float height, float linewidth, float antialias)
95.             {
96.                 float w = linewidth / 2.0 + antialias;
97.                 float2 start = -float2(body / 2.0, 0.0);
98.                 float2 end = float2(body / 2.0, 0.0);
99.  
100.                 // Head: 3 lines
101.                 float d1 = DistanceToLine(texcoord, end, end - head * float2(1.0, -height));
102.                 float d2 = DistanceToLine(texcoord, end - head * float2(1.0, height), end);
103.                 float d3 = texcoord.x - end.x + head;
104.  
105.                 // Body: 1 segment
106.                 float d4 = DistanceToSegment(texcoord, start, end - float2(linewidth, 0.0));
107.  
108.                 float d = min(max(max(d1, d2), -d3), d4);
109.                 return d;
110.             }
111.  
112.             #define PI 3.14159265359
113.             float4 frag (v2f i) : SV_Target
114.             {
115.                 float motionVectMin = -1;
116.                 float motionVectMax = 1;
117.              
118.                 float rgMin = 0;
119.                 float rgMax = 1;
120.  
121.                 float motionVectorRange =  motionVectMax - motionVectMin;
122.                 float rgRange =  rgMax - rgMin;
123.  
124.                 float2 mv = SampleMotionVectors(i.uv);
125.      
126.                 _MaxX = 1.3f; //    max(_MaxX, mv.x);
127.                 _MaxY = max(_MaxY, mv.y);
128.              
129.                 _MinX = min(_MinX, mv.x);
130.                 _MinY = min(_MinY, mv.y);
131.  
132.                 // Background color intensity - keep this low unless you want to make your eyes bleed
133.                 const float kMinIntensity = 0.03f;
134.                 const float kMaxIntensity = 0.50f;
135.  
136.                 // Map motion vector direction to color wheel (hue between 0 and 360deg)
137.                 float phi = atan2(mv.x, mv.y);
138.                 float hue = (phi / PI + 1.0) * 0.5;
139.                 float r = abs(hue * 6.0 - 3.0) - 1.0;
140.                 float g = 2.0 - abs(hue * 6.0 - 2.0);
141.                 float b = 2.0 - abs(hue * 6.0 - 4.0);
142.      
143.                 float maxSpeed = 60.0f / 0.15f; // Admit that 15% of a move the viewport by second at 60 fps is really fast
144.                 float absoluteLength = saturate(length(mv.xy) * maxSpeed);
145.                 float3 color = float3(r, g, b) * lerp(kMinIntensity, kMaxIntensity, absoluteLength);
146.                 color = saturate(color);
147.  
148.                 if (!_drawArrows)
149.                     return float4(color, 1);
150.  
151.                 // Grid subdivisions - should be dynamic
152.                 const float kGrid = 64.0;
153.  
154.                 float arrowSize = 500;
155.                 float4 screenSize = float4(arrowSize, arrowSize, 1.0 / arrowSize, 1.0 / arrowSize);
156.  
157.                 // Arrow grid (aspect ratio is kept)
158.                 float aspect = screenSize.y * screenSize.z;
159.                 float rows = floor(kGrid * aspect);
160.                 float cols = kGrid;
161.                 float2 size = screenSize.xy / float2(cols, rows);
162.                 float body = min(size.x, size.y) / sqrt(2.0);
163.                 float2 positionSS = i.uv;
164.                 positionSS *= screenSize.xy;
165.                 float2 center = (floor(positionSS / size) + 0.5) * size;
166.                 positionSS -= center;
167.  
168.                 // Sample the center of the cell to get the current arrow vector
169.                 float2 mv_arrow = 0.0f;
170. #if DONT_USE_NINE_TAP_FILTER
171.                 mv_arrow = SampleMotionVectors(center * screenSize.zw);
172. #else
173.                 for (int i = -1; i <= 1; ++i) for (int j = -1; j <= 1; ++j)
174.                     mv_arrow += SampleMotionVectors((center + float2(i, j)) * screenSize.zw);
175.                 mv_arrow /= 9.0f;
176. #endif
177.                 mv_arrow.y *= -1;
178.  
179.                 // Skip empty motion
180.                 float d = 0.0;
181.                 if (any(mv_arrow))
182.                 {
183.                     // Rotate the arrow according to the direction
184.                     mv_arrow = normalize(mv_arrow);
185.                     float2x2 rot = float2x2(mv_arrow.x, -mv_arrow.y, mv_arrow.y, mv_arrow.x);
186.                     positionSS = mul(rot, positionSS);
187.  
188.                     d = DrawArrow(positionSS, body, 0.25 * body, 0.5, 2.0, 1.0);
189.                     d = 1.0 - saturate(d);
190.                 }
191.  
192.                 // Explicitly handling the case where mv == float2(0, 0) as atan2(mv.x, mv.y) above would be atan2(0,0) which
193.                 // is undefined and in practice can be incosistent between compilers (e.g. NaN on FXC and ~pi/2 on DXC)
194.                 if(!any(mv))
195.                     color = float3(0, 0, 0);
196.                  
197.                 return float4(color + d.xxx, 1);
198.             }
199.             ENDCG
200.         }
201.     }
202. }
203.  

But I am having trouble retrieving the min max values. I will post a question for it in the shaders thread.

 

Ok I am back to your suggestions here is that I came up with but all it outputs is blue Do you have any thoughts for me?

Code (CSharp):

1. public class MotionVectorManager : MonoBehaviour
2. {
3.     int frameNumber = 0;
4.  
5.     public void Start()
6.     {
7.         RenderPipelineManager.endCameraRendering += OnEndCameraRendering;
8.     }
9.  
10.     void OnEndCameraRendering(ScriptableRenderContext context, Camera camera)
11.     {
12.         Texture vectorMotionTex = Shader.GetGlobalTexture("_CameraMotionVectorsTexture");
13.         SaveTextureToFile(vectorMotionTex, "C:/tmp/Image" + frameNumber++ + ".exr");
14.     }
15.  
16.     static public void SaveTextureToFile(Texture source, string filePath, bool asynchronous = true, System.Action<bool> done = null)
17.     {
18.         if (!(source is Texture2D || source is RenderTexture))
19.         {
20.             done?.Invoke(false);
21.             return;
22.         }
23.  
24.         // resize the original image:
25.         var resizeRT = RenderTexture.GetTemporary(source.width, source.height, 0, RenderTextureFormat.RGHalf);
26.         Graphics.Blit(source, resizeRT);
27.  
28.         // create a native array to receive data from the GPU:
29.         var narray = new NativeArray<byte>(source.width * source.height * 4, Allocator.Persistent, NativeArrayOptions.UninitializedMemory);
30.  
31.         // request the texture data back from the GPU:
32.         var request = AsyncGPUReadback.RequestIntoNativeArray(ref narray, resizeRT, 0, (AsyncGPUReadbackRequest request) =>
33.         {
34.             // if the readback was successful, encode and write the results to disk
35.             if (!request.hasError)
36.             {
37.                 NativeArray<byte> encoded;
38.                 encoded = ImageConversion.EncodeNativeArrayToEXR(narray, resizeRT.graphicsFormat, (uint)source.width, (uint)source.height);
39.                 System.IO.File.WriteAllBytes(filePath, encoded.ToArray());
40.                 encoded.Dispose();
41.             }
42.  
43.             narray.Dispose();
44.  
45.             // notify the user that the operation is done, and its outcome.
46.             done?.Invoke(!request.hasError);
47.         });
48.  
49.         if (!asynchronous)
50.             request.WaitForCompletion();
51.     }
52. }
53.  

I know the motion vectors are being generated because the above Shader worked. But I need to get the min max information from the data and I can't figure out how to pass that information from the Shader to the CPU. So I tried the above.

 

Never mind the above actually works ! I just must have the format incorrect. The background is all blue and the motion information is there but very very faint
Well at least I have something! Thank you @c0d3_m0nk3y I owe you.