Transparent Window using ChromaKey shader not working

Hi

I am trying to create a transparent window for my unity project to run on Windows such that the background of the scene is transparent, showing the desktop below, and the objects in the scene are rendered opaquely on top.

I found some great articles which contained the following TransparentWindow script, which as instructed, I attached to my main camera

Code (CSharp):

1. using System;
2. using System.Runtime.InteropServices;
3. using UnityEngine;
4.  
5. public class TransparentWindow : MonoBehaviour
6. {
7.     [SerializeField]
8.     private Material m_Material;
9.  
10.     private struct MARGINS
11.     {
12.         public int cxLeftWidth;
13.         public int cxRightWidth;
14.         public int cyTopHeight;
15.         public int cyBottomHeight;
16.     }
17.  
18.     [DllImport("user32.dll")]
19.     private static extern IntPtr GetActiveWindow();
20.  
21.     [DllImport("user32.dll")]
22.     private static extern int SetWindowLong(IntPtr hWnd, int nIndex, uint dwNewLong);
23.  
24.     [DllImport("user32.dll")]
25.     static extern bool ShowWindowAsync(IntPtr hWnd, int nCmdShow);
26.  
27.     [DllImport("user32.dll", EntryPoint = "SetLayeredWindowAttributes")]
28.     static extern int SetLayeredWindowAttributes(IntPtr hwnd, int crKey, byte bAlpha, int dwFlags);
29.  
30.     [DllImport("user32.dll", EntryPoint = "SetWindowPos")]
31.     private static extern int SetWindowPos(IntPtr hwnd, int hwndInsertAfter, int x, int y, int cx, int cy, int uFlags);
32.  
33.     [DllImport("Dwmapi.dll")]
34.     private static extern uint DwmExtendFrameIntoClientArea(IntPtr hWnd, ref MARGINS margins);
35.  
36.     const int GWL_STYLE = -16;
37.     const uint WS_POPUP = 0x80000000;
38.     const uint WS_VISIBLE = 0x10000000;
39.     const int HWND_TOPMOST = -1;
40.  
41.     void Start()
42.     {
43.         #if !UNITY_EDITOR // You really don't want to enable this in the editor..
44.  
45.         int fWidth = Screen.width;
46.         int fHeight = Screen.height;
47.         var margins = new MARGINS() { cxLeftWidth = -1 };
48.         var hwnd = GetActiveWindow();
49.  
50.         SetWindowLong(hwnd, GWL_STYLE, WS_POPUP | WS_VISIBLE);
51.  
52.         // Transparent windows with click through
53.         SetWindowLong(hwnd, -20, 524288 | 32);//GWL_EXSTYLE=-20; WS_EX_LAYERED=524288=&h80000, WS_EX_TRANSPARENT=32=0x00000020L
54.         SetLayeredWindowAttributes(hwnd, 0, 255, 0);// Transparency=51=20%, LWA_ALPHA=2
55.         SetWindowPos(hwnd, HWND_TOPMOST, 0, 0, fWidth, fHeight, 32 | 64); //SWP_FRAMECHANGED = 0x0020 (32); //SWP_SHOWWINDOW = 0x0040 (64)
56.         DwmExtendFrameIntoClientArea(hwnd, ref margins);
57.  
58.         #endif
59.     }
60.  
61.     void OnRenderImage(RenderTexture from, RenderTexture to)
62.     {
63.         Graphics.Blit(from, to, m_Material);
64.     }
65. }

I added a shader as follows, to convert pixels of a given colour to transparent pixels, which I assigned to a material

Code (CSharp):

1. // Upgrade NOTE: replaced 'mul(UNITY_MATRIX_MVP,*)' with 'UnityObjectToClipPos(*)'
2.  
3. Shader "Custom/ChromakeyTransparent" {
4.     Properties{
5.         _MainTex("Base (RGB)", 2D) = "white" {}
6.         _TransparentColourKey("Transparent Colour Key", Color) = (0,0,0,1)
7.         _TransparencyTolerance("Transparency Tolerance", Float) = 0.01
8.     }
9.         SubShader{
10.             Pass {
11.                 Tags { "RenderType" = "Opaque" }
12.                 LOD 200
13.  
14.                 CGPROGRAM
15.  
16.                 #pragma vertex vert
17.                 #pragma fragment frag
18.                 #include "UnityCG.cginc"
19.  
20.                 struct a2v
21.                 {
22.                     float4 pos : POSITION;
23.                     float2 uv : TEXCOORD0;
24.                 };
25.  
26.                 struct v2f
27.                 {
28.                     float4 pos : SV_POSITION;
29.                     float2 uv : TEXCOORD0;
30.                 };
31.  
32.                 v2f vert(a2v input)
33.                 {
34.                     v2f output;
35.                     output.pos = UnityObjectToClipPos(input.pos);
36.                     output.uv = input.uv;
37.                     return output;
38.                 }
39.  
40.                 sampler2D _MainTex;
41.                 float3 _TransparentColourKey;
42.                 float _TransparencyTolerance;
43.  
44.                 float4 frag(v2f input) : SV_Target
45.                 {
46.                     // What is the colour that *would* be rendered here?
47.                     float4 colour = tex2D(_MainTex, input.uv);
48.  
49.                     // Calculate the different in each component from the chosen transparency colour
50.                     float deltaR = abs(colour.r - _TransparentColourKey.r);
51.                     float deltaG = abs(colour.g - _TransparentColourKey.g);
52.                     float deltaB = abs(colour.b - _TransparentColourKey.b);
53.  
54.                     // If colour is within tolerance, write a transparent pixel
55.                     if (deltaR < _TransparencyTolerance && deltaG < _TransparencyTolerance && deltaB < _TransparencyTolerance)
56.                     {
57.                        
58.                         return float4(0.0f, 0.0f, 0.0f, 0.0f);
59.                     }
60.  
61.                     // Otherwise, return the regular colour
62.                     return colour;
63.                 }
64.                 ENDCG
65.             }
66.         }
67. }

I set the camera Clear Flags to Solid Color and the Background color to pure green (00FF00)
On the material which has the shader assigned I set the Transparent Colour Key to the same green (00FF00)
I assigned the material to the Camera's TransparentWindow script "Material" property
I can see that the shader is doing its job and the green background becomes black when I run the code in the editor.
When I run the code from "build and run" I am expecting the black pixels to become transparent but they remain black
If I play with the settings being passed to SetLayeredWindowAttributes to change the 255 to 100, for example, the window becomes partially transparent as expected.
So I know the shader works and I know I can make a window transparent but I cannot seem to get the two to work together.
I am using Unity Personal 2020.1.0f1

Any help much appreciated

 

Answering my own question... found this article: https://forum.unity.com/threads/window-as-transparent-overlay-not-working-since-2018-3-0b12.593653/

Seems there was a change in more recent versions of Unity that requires this checkbox to be unchecked in order for transparent windows to work, its checked by default:

Build Settings -> Player Settings -> Player -> Resolution and Presentation -> Use DXGI Flip Model Swapchain for D3D11