Bounding boxes around heads only of characters using Skinned Mesh Renderers

Hi,

We currently do not have support for this in the package. That said, you may be able to achieve this with some trickery and shader work. Internally, the bounding box labeler uses an instance segmentation image to decide which pixels in the rendered image belong to each object. This is done using the shader file named InstanceSegmentation.shader. This shader is fairly simple. It just outputs one color for all the output pixels of a rendered object.

To have the bounding box around only the heads, you would need to alter this shader to produce a 0 output for all pixels that do not belong to the head area and output the normal color for the pixels that do belong to the head. This can be done by passing in a texture as a mask. This texture would be comprised of two colors to separate the head from the rest of the body. For example, the parts that would map to the head could be white and the rest could be black.

The possibly tricky part is that you need to generate mask textures for all your body meshes with proper UVs. Then in Unity, you can add a simple component to your human prefabs and all this component does is hold a reference to a texture, which is the prefab's corresponding head mask.

Let's say the component is named HeadMaskTexture. You would need to add this code to InstanceSegmentationCrossPipelinePass.cs at line 74 to pass the texture to the above mentioned shader:

Code (CSharp):

1. var maintex = renderer.gameObject.GetComponent<HeadMaskTexture>().maskTexture;
2. if (maintex != null)
3.     mpb.SetTexture("_MainTex", maintex);

And modify the shader code to treat black pixels differently:

Code (CSharp):

1. Shader "Perception/InstanceSegmentation"
2. {
3.     Properties
4.     {
5.         [PerObjectData] _SegmentationId("Segmentation ID", vector) = (0,0,0,1)
6.         _MainTex ("Main Texture", 2D) = "white" { }
7.     }
8.     SubShader
9.     {
10.         Tags { "RenderType"="Opaque" }
11.         LOD 100
12.  
13.  
14.         Pass
15.         {
16.             //Tags { "LightMode" = "SRP" }
17.  
18.             CGPROGRAM
19.             #pragma vertex vert
20.             #pragma fragment frag
21.  
22.             #include "UnityCG.cginc"
23.             #include "Packing.hlsl"
24.  
25.             float4 _SegmentationId;
26.             sampler2D _MainTex;
27.             float4 _MainTex_ST;
28.  
29.             struct appdata
30.             {
31.                 float4 vertex : POSITION;
32.                 float2 uv : TEXCOORD0;
33.             };
34.  
35.             struct v2f
36.             {
37.                 float4 vertex : SV_POSITION;
38.                 float2 uv : TEXCOORD0;
39.             };
40.  
41.             v2f vert (appdata v)
42.             {
43.                 v2f o;
44.                 o.vertex = UnityObjectToClipPos(v.vertex);
45.                 o.uv = TRANSFORM_TEX(v.uv, _MainTex);
46.                 return o;
47.             }
48.  
49.             fixed4 frag (v2f i) : SV_Target
50.             {
51.                 // sample the texture
52.                 fixed4 col = tex2D(_MainTex, i.uv);
53.                 //if the pixel is black, output 0, which means removing this pixel from the instance segmentation image (and thus bounding box2d)
54.                 if (col.r == 0 && col.g == 0 && col.b == 0)
55.                     return fixed4(0, 0, 0, 0);
56.  
57.                 return _SegmentationId;
58.             }
59.             ENDCG
60.         }
61.     }
62. }
63.