Question Ryan Brucks Improved UV Dilation Unity Implementation

In the quest to fix seam lighting with normal maps, I found Ryan Brucks had done an apparently really effective solution, but my port of that is bugged.

Can you cast your eyes over this and spot where the problem might be? I'm getting white spots and a generally bad result and it's probably something I missed when porting.

Original Code:

Code (CSharp):

1. //////////////// UV Positional Dilation ///////////////////////////
2. //** Tex **// Input Texture Object storing Volume Data
3. //** UV **// Input float2 for UVs
4. //** TextureSize **// Resolution of render target
5. //** MaxSteps **// Pixel Radius to search
6.  
7.  
8. float texelsize = 1 / TextureSize;
9. float mindist = 10000000;
10. float2 offsets[8] = {float2(-1,0), float2(1,0), float2(0,1), float2(0,-1), float2(-1,1), float2(1,1), float2(1,-1), float2(-1,-1)};
11.  
12. float3 sample = Tex.SampleLevel(TexSampler, UV, 0);
13. float3 curminsample = sample;
14.  
15. if(sample.x == 0 && sample.y == 0 && sample.z == 0)
16. {
17.     int i = 0;
18.     while(i < MaxSteps)
19.     {
20.         i++;
21.         int j = 0;
22.         while (j < 8)
23.         {
24.             float2 curUV = UV + offsets[j] * texelsize * i;
25.             float3 offsetsample = Tex.SampleLevel(TexSampler, curUV, 0);
26.  
27.             if(offsetsample.x != 0 || offsetsample.y != 0 || offsetsample.z != 0)
28.             {
29.                 float curdist = length(UV - curUV);
30.  
31.                 if (curdist < mindist)
32.                 {
33.                     float2 projectUV = curUV + offsets[j] * texelsize * i * 0.25;
34.                     float3 direction = Tex.SampleLevel(TexSampler, projectUV, 0);
35.                     mindist = curdist;
36.  
37.                     if(direction.x != 0 || direction.y != 0 || direction.z != 0)
38.                     {
39.                         float3 delta = offsetsample - direction;
40.                         curminsample = offsetsample + delta * 4
41.                     }
42.  
43.                    else
44.                     {
45.                         curminsample = offsetsample;
46.                     }
47.                 }
48.             }
49.             j++;
50.         }
51.     }
52. }
53.  
54. return curminsample;

My port:

Code (CSharp):

1. Shader "Internal/Dilate"
2. {
3.     Properties
4.     {
5.         _MainTex ("Texture", 2D) = "white" {}
6.     }
7.  
8.     SubShader
9.     {
10.         Tags { "RenderType"="Opaque" }
11.         LOD 100
12.  
13.         Pass
14.         {
15.             CGPROGRAM
16.             #pragma vertex vert
17.             #pragma fragment frag
18.  
19.             #include "UnityCG.cginc"
20.  
21.             struct appdata
22.             {
23.                 float4 vertex : POSITION;
24.                 float2 uv : TEXCOORD0;
25.             };
26.  
27.             struct v2f
28.             {
29.                 float2 uv : TEXCOORD0;
30.                 float4 vertex : SV_POSITION;
31.             };
32.  
33.             sampler2D _MainTex;
34.             float _TexelSize;
35.             int _MaxSteps;
36.  
37.             float3 Sample(float2 uv, float lod)
38.             {
39.                 return tex2Dlod(_MainTex, float4(uv, 0, lod)).rgb;
40.             }
41.  
42.             float3 SimpleDilate(float2 uv)
43.             {  
44.                 float2 offsets[8] = { float2(-1,0), float2(1,0), float2(0,1), float2(0,-1), float2(-1,1), float2(1,1), float2(1,-1), float2(-1,-1) };
45.                 float minDist = 10000000;
46.                 float3 samp = Sample(uv, 0);
47.                 float3 currentMinSample = samp;
48.  
49.                 if (samp.x == 0 && samp.y == 0 && samp.z == 0)
50.                 {
51.                     int i = 0;
52.                     while (i < _MaxSteps)
53.                     {
54.                         i++;
55.                         int j = 0;
56.                         while (j < 8)
57.                         {
58.                             float2 offsetUV = uv + offsets[j] * _TexelSize * i;
59.                             float3 offsetSample = Sample(offsetUV, 0);
60.  
61.                             if (offsetSample.x != 0 || offsetSample.y != 0 || offsetSample.z != 0)
62.                             {
63.                                 float dist = length(uv - offsetUV);
64.                                 if (dist <= minDist)
65.                                 {
66.                                     minDist = dist;
67.                                     float2 extrapolatedUV = offsetUV + offsets[j] *_TexelSize * i * 0.25;
68.                                     float3 direction = Sample(extrapolatedUV, 0);
69.  
70.                                     if (direction.x != 0 || direction.y != 0 || direction.z != 0)
71.                                     {
72.                                         float3 delta = offsetSample - direction;
73.                                         currentMinSample = offsetSample + delta * 4;
74.                                     }
75.                                     else
76.                                     {
77.                                         currentMinSample = offsetSample;
78.                                     }
79.                                 }
80.                             }
81.                             j++;
82.                         }
83.                     }
84.                 }
85.  
86.                 return currentMinSample;
87.             }
88.  
89.             v2f vert (appdata v)
90.             {
91.                 v2f o;
92.                 o.uv = v.uv;
93.                 o.vertex = UnityObjectToClipPos(v.vertex);
94.                 return o;
95.             }
96.  
97.             float4 frag(v2f i) : SV_Target
98.             {
99.                 return float4(SimpleDilate(i.uv), 1);
100.             }
101.             ENDCG
102.         }
103.     }
104. }

Original article and explanation:
https://shaderbits.com/blog/uv-dilation

Thanks for any insights

 

I'm basically assuming linear textures here, for both UE4 and Unity, but I don't know if somehow the texels are wrong or, it just seems odd that it wouldn't work as expected since the code is so simple. Could it be a difference in how UE4 vs Unity sample textures? Any suggestions will help at this point I think.

 

Seems to work okay? What results are you getting?



The breakthrough was when I realized it probably needs higher precision source, since if it's extrapolating, lack of precision gets amplified and makes results look a bit weird.

 

Hmmm nevermind, with weirder black gaps it does produce white texels for me too.



Although, even in one of his examples there are some weird white artifacts as well, (see : https://storage.googleapis.com/wzukusers/user-22455410/images/57db548291436yaCoBNI/DilatedBP.jpg ), so maybe it's a byproduct of the method?

Brain's too tired to figure out if the white artifacts somehow make sense.

EDIT: I wonder if you switched it to using an alpha channel instead of pure black pixels if it would improve things.

 

Yeah, I don't think you got anything wrong. I think his original technique is mildly flawed. For some use cases it might produce slightly better results than the traditional dilate, and possibly worse in others. Even his "look how much better this looks" example is still clearly not right. Better, but not right.

The real solution to those kinds of UV seam issues would be something that actually looks at the texture on the other side of the seam and tries to fit the two edges together.
http://miciwan.com/SIGGRAPH2013/Lighting Technology of The Last Of Us.pdf
https://www.sebastiansylvan.com/post/LeastSquaresTextureSeams/
I've seen more on this topic before, but I can't track down the links for some reason. However it should also be called out that Unity already has this functionality, at least for its own Progressive Lightmapper lightmaps.
https://docs.unity3d.com/Manual/Lightmapping-SeamStitching.html
Too bad this doesn't seem to be exposed anywhere as something you could just call on a mesh w/ an existing texture. Though it those techniques would totally mess up tangent space normal maps. Really you'd want to convert to world space normals, do the edge fixup, and then transform back into tangent space, which gets tricky when you're talking about the tangent space outside of a triangle's bounds, and since you may round out intentional hard edges.

 

This sounds like a horror story ahead of me to be honest...

 

Some more resources:

https://github.com/zfergus/seam-erasure-js
(hot as hell results but thoughts? they claim normals would be OK)

https://github.com/qiankanglai/seamoptimizer
(a unity implementation of the last of us approach, untested though)

Having a think about those or doing some experimental compute shader lark. Does seem a shame we can't get the pixels in other uv islands based on some tangent traversal... then blend them.

 

I would assume world space normal maps are being used, in which case it'd totally work.

The main thing to realize is the areas outside of the UV'd area might be ugly, and that's totally okay. There's an undue focus on making sure the dilated texture itself looks nice and is free of hard edges. If you look at a lightmap that's had some forms of seam correction applied, it can look like total chaos in the areas outside of the triangle bounds. Weird bright and dark spots that are clearly wrong ... but actually aren't. They exist because they're there to get the interpolated values on the seam itself to match up, not to try to get the texture that you'll never see to look nice. The papers with those techniques were the ones I was thinking of for this topic rather than the two I linked to. Not sure where the ones I was thinking of are unfortunately. I don't think they weren't explicitly on the topic of lightmap seams, but rather general UV seams when doing 3D painting.

All that said, those bright spots in the above are actually wrong. Though I can't figure out what the cause is just running the code over in my head.

 

The bright spots get reduced (although some weirdness stays) if I switch the original image to point filtering (although then the result has a lot more banding), which makes me think that with bilinear, it somehow samples a value that is interpolated towards black a bit, so it extrapolates much more than it needs to...? Or something like that.

This seems to be based on this : https://cragl.cs.gmu.edu/seamless/

From a quick glance, it seems like it touches a lot more texels than just the ones around the seams, so it's like multiple steps further than the least squares method. Examples look good though.

Although I feel a bit weird on how much different the "fixed" version of this normal map is (at around 0:38):

 

Yeah but the seam looks a bit more weird than not so I'd probably go for that. They have a python version and Unity can run python scripts... maybe it's just safer porting the js version though since I've 0 python expertise and would just cock it up completely.

Re: white spots, I also tried your suggestion of storing in .a but that led to absolutely no difference as you can imagine.

 

Do the following:

Switch the masking to .a

But then instead of checking if (sample.a == 0)

do

if (sample.a < 1.0)

and also change the two following if statements to

if (offsetSample.a == 1)

if (direction.a == 1)

This makes it so that it rejects values that are interpolated towards black even in the slightest.

This improved things a lot for me (although some artifacts still remain, the one on the top left is quite visible, but maybe this pushes it into "good enough" territory?)

 

In case anyone is interested I also wrote a simple editor script that writes the dilated results to a *****_dilated.png file.

You can find it under Tools / Dilate Texture

 

I tried your tweaks and they do help! I think this thread and your latest package will help others too, so thank you! Very cool of you to add a package.

As for me, it helps for runtime texture edits, but for the final game I'm going to have to at some point implement a proper (slow) edge blend on the editor side. A shame Unity doesn't expose code that already does this.