# Question Calculate Tangent Space Data In C# Script

Discussion in 'General Graphics' started by Jahvan, Jan 12, 2021.

1. ### Jahvan

Joined:
Feb 1, 2017
Posts:
10
I'm creating a system that changes the shape of a mesh using a shader. It aims to achieve the same result as what the SkinnedMeshRenderer does with a Blend Shape. The mesh altering data is stored in 3 texture maps. There's one for vertex offset, one for normals, and one for tangents.

Everything works great when the data is in Object Space and the mesh is static:

Sadly, this solution isn't good enough for skinned meshes. Once the mesh leaves bindpose, everything breaks. I imagine the issue is similar to having an object space normal map on an animated model. To which the solution is instead to use a tangent space normal map.

So, to the best of my knowledge, I altered my script and shader to handle tangent space data. But it seems I am out of my depth here. And I can't seem to find any information relating to this online.

Here's my naive method for converting the data to tangent space, which I believe is the root of the problem:

Code (CSharp):
1.
2.         Vector3[] norms = mesh.normals;
3.         Vector4[] tans = mesh.tangents;
4.
5.         for(int a = 0; a < mesh.vertexCount; a++)
6.         {
7.
8.             Vector3 OS_normal = norms[a];
9.             Vector3 OS_tangent = new Vector3(tans[a].x, tans[a].y, tans[a].z);
10.
11.             Vector3 OS_binormal = Vector3.Cross(OS_normal, OS_tangent) * tans[a].w;
12.
13.             Vector3 ToTangentSpace(Vector3 vec)
14.             {
15.
16.                 Quaternion toTangentSpace = Quaternion.LookRotation(Vector3.forward, Vector3.up) * Quaternion.Inverse(Quaternion.LookRotation(OS_normal, OS_binormal));
17.
19.
20.             }
21.

The result:

And here's the full script for good measure:

Code (CSharp):
1.
2. using UnityEngine;
3.
4. public class TextureShapeTest : MonoBehaviour
5. {
6.
7.     public SkinnedMeshRenderer renderer;
8.
9.     public Material material;
10.
11.     public string shapeName;
12.
13.     public float maxPositionOffset = 2;
14.
15.     public float maxNormalOffset = 2;
16.
17.     public enum Axis
18.     {
19.
20.         X, Y, Z, InvertedX, InvertedY, InvertedZ
21.
22.     }
23.
24.     public Axis RChannelAxis = Axis.X;
25.     public Axis GChannelAxis = Axis.Y;
26.     public Axis BChannelAxis = Axis.Z;
27.
28.     public Texture2D position, normal, tangent;
29.
30.     void Start()
31.     {
32.
33.         Mesh mesh = renderer.sharedMesh;
34.
35.         BlendShape shape = null;
36.
37.         for(int a = 0; a < mesh.blendShapeCount; a++)
38.         {
39.
40.             string name = mesh.GetBlendShapeName(a);
41.
42.             if (name == shapeName)
43.             {
44.
45.                 shape = new BlendShape(mesh, name); // Custom class for extracting/storing blend shape data
46.
47.                 break;
48.
49.             }
50.
51.         }
52.
53.         if (shape == null) return;
54.
55.         int textureSize = 32;
56.
57.         while (textureSize * textureSize < mesh.vertexCount) textureSize = textureSize * 2;
58.
59.         int pixelCount = textureSize * textureSize;
60.
61.         Color[] positionPixels = new Color[pixelCount];
62.         Color[] normalPixels = new Color[pixelCount];
63.         Color[] tangentPixels = new Color[pixelCount];
64.
65.         Color clear = Color.clear;
66.
67.         for(int a = 0; a < pixelCount; a++)
68.         {
69.
70.             positionPixels[a] = clear;
71.             normalPixels[a] = clear;
72.             tangentPixels[a] = clear;
73.
74.         }
75.
76.         position = new Texture2D(textureSize, textureSize, TextureFormat.RGBAFloat, false, true);
77.         normal = new Texture2D(textureSize, textureSize, TextureFormat.RGBAFloat, false, true);
78.         tangent = new Texture2D(textureSize, textureSize, TextureFormat.RGBAFloat, false, true);
79.
80.         position.filterMode = normal.filterMode = tangent.filterMode = FilterMode.Point;
81.
82.         BlendShape.Frame frame = shape.frames[0];
83.
84.         Vector3[] norms = mesh.normals;
85.         Vector4[] tans = mesh.tangents;
86.
87.         for(int a = 0; a < mesh.vertexCount; a++)
88.         {
89.
90.             Vector3 OS_normal = norms[a];
91.             Vector3 OS_tangent = new Vector3(tans[a].x, tans[a].y, tans[a].z);
92.
93.             Vector3 OS_binormal = Vector3.Cross(OS_normal, OS_tangent) * tans[a].w;
94.
95.             Vector3 ToTangentSpace(Vector3 vec)
96.             {
97.
98.                 Quaternion toTangentSpace = Quaternion.LookRotation(Vector3.forward, Vector3.up) * Quaternion.Inverse(Quaternion.LookRotation(OS_normal, OS_binormal));
99.
101.
102.             }
103.
104.             float SelectDelta(Vector3[] deltas, Axis axis)
105.             {
106.
107.                 switch(axis)
108.                 {
109.
110.                     case Axis.X:
112.                     case Axis.Y:
114.                     case Axis.Z:
116.                     case Axis.InvertedX:
118.                     case Axis.InvertedY:
120.                     case Axis.InvertedZ:
122.
123.                 }
124.
125.                 return 0;
126.
127.             }
128.
129.             float SelectPosition(Axis axis) { return SelectDelta(frame.deltaVertices, axis); }
130.             float SelectNormal(Axis axis) { return SelectDelta(frame.deltaNormals, axis); }
131.             float SelectTangent(Axis axis) { return SelectDelta(frame.deltaTangents, axis); }
132.
133.             Color pix_Position = new Color(
134.                 (Mathf.Clamp(SelectPosition(RChannelAxis) / maxPositionOffset, -1, 1) + 1) / 2f,
135.                 (Mathf.Clamp(SelectPosition(GChannelAxis) / maxPositionOffset, -1, 1) + 1) / 2f,
136.                 (Mathf.Clamp(SelectPosition(BChannelAxis) / maxPositionOffset, -1, 1) + 1) / 2f, 0.5f);
137.
138.             Color pix_Normal = new Color(
139.                 (Mathf.Clamp(SelectNormal(RChannelAxis) / maxNormalOffset, -1, 1) + 1) / 2f,
140.                 (Mathf.Clamp(SelectNormal(GChannelAxis) / maxNormalOffset, -1, 1) + 1) / 2f,
141.                 (Mathf.Clamp(SelectNormal(BChannelAxis) / maxNormalOffset, -1, 1) + 1) / 2f, 0.5f);
142.
143.             Color pix_Tangent = new Color(
144.                 (Mathf.Clamp(SelectTangent(RChannelAxis) / maxNormalOffset, -1, 1) + 1) / 2f,
145.                 (Mathf.Clamp(SelectTangent(GChannelAxis) / maxNormalOffset, -1, 1) + 1) / 2f,
146.                 (Mathf.Clamp(SelectTangent(BChannelAxis) / maxNormalOffset, -1, 1) + 1) / 2f, 0.5f);
147.
148.             positionPixels[a] = pix_Position;
149.             normalPixels[a] = pix_Normal;
150.             tangentPixels[a] = pix_Tangent;
151.
152.         }
153.
154.         position.SetPixels(positionPixels);
155.         normal.SetPixels(normalPixels);
156.         tangent.SetPixels(tangentPixels);
157.
158.         position.Apply();
159.         normal.Apply();
160.         tangent.Apply();
161.
162.         material.SetTexture("_PositionMap", position);
163.         material.SetTexture("_NormalMap", normal);
164.         material.SetTexture("_TangentMap", tangent);
165.
166.         material.SetFloat("_MaxPositionOffset", maxPositionOffset);
167.         material.SetFloat("_MaxNormalOffset", maxNormalOffset);
168.
169.         renderer.sharedMaterial = material;
170.
171.     }
172.
173. }
174.