Grayed out renders when using custom projection matrix

I am having a weird problem with my rendering after setting a custom projection matrix. If the rendering mode is set to Forward then I get a weird box where the scene is rendered properly. Outside of this box the scene is grayed out almost as if it were not receiving any light. The custom projection matrix is for an asymetrical viewing frustum to do the holographic projection thing. The weird box moves around as the camera moves and seems to get worse the further the camera is from the objects. If I set the rendering mode to Per Vertex it seems to work fine. Is there an option to set or am I messing something up?


Render with box in forward rendering mode
https://www.dropbox.com/sh/itxp1s0j905d6vl/wyhCjvzNEA/Unity3d/WhiteSquare1.jpg

More or less proper render with vertex only
https://www.dropbox.com/sh/itxp1s0j905d6vl/eWW_OTrZ9Z/Unity3d/WhiteSquare2.jpg

Scene view
https://www.dropbox.com/sh/itxp1s0j905d6vl/oHU9kkFMaV/Unity3d/WhiteSquare3.jpg

Here is the code:

Code (csharp):

1. using UnityEngine;
2. using System.Collections;
3. using System.Net.Sockets;
4. using System.Net;
5. using System;
6. using System.Runtime.InteropServices;
7.  
8. public class HeadTrackingCamera : MonoBehaviour
9. {
10.     public bool enable = false;
11.     UdpClient udpClient;
12.     IPEndPoint endPoint;
13.     Vector3 headPosition;
14.     Vector3 sensorOffset;
15.  
16.     Vector3 pa, pb, pc;
17.     Vector3 vr, vu, vn;
18.     float n, f;
19.  
20.     // Use this for initialization
21.     void Start ()
22.     {
23.         Application.runInBackground = true;
24.  
25.         enable = true;
26.  
27.         float screenWidth = 0.406f; //Dimensions are in meters
28.         float screenHeight = 0.254f;
29.         float halfWidth = screenWidth / 2;
30.         float halfHeight = screenHeight / 2;
31.  
32.         n = 0.05f; //what is the right value for this??????
33.         f = 100; //what is the right value for this?????
34.         headPosition = new Vector3(0.0f, -0.005f, -3.14f);
35.         sensorOffset = new Vector3(-0.08f, 0.17f, 0.14f);
36.  
37.         pa = new Vector3(-halfWidth, -halfHeight, 0);
38.         pb = new Vector3(halfWidth, -halfHeight, 0);
39.         pc = new Vector3(-halfWidth, halfHeight, 0);
40.  
41.         vr = new Vector3(1, 0, 0);
42.         vu = new Vector3(0, 1, 0);
43.         vn = new Vector3(0, 0, -1);
44.  
45.         IPAddress ip = IPAddress.Parse("127.0.0.1");
46.         endPoint = new IPEndPoint(ip, 11000);
47.         udpClient = new UdpClient(endPoint);
48.         listenForMessages();
49.     }
50.  
51.     void listenForMessages()
52.     {
53.         udpClient.BeginReceive(new AsyncCallback(messageRecieved), null);
54.     }
55.  
56.     void messageRecieved(IAsyncResult ar)
57.     {
58.         Byte[] data = udpClient.EndReceive(ar, ref endPoint);
59.         int size = Marshal.SizeOf(headPosition);
60.         IntPtr ptr = Marshal.AllocHGlobal(size);
61.         Marshal.Copy(data, 0, ptr, size);
62.         headPosition = (Vector3)Marshal.PtrToStructure(ptr, typeof(Vector3)) - sensorOffset;
63.         Marshal.FreeHGlobal(ptr);
64.  
65.         //Debug.Log(headPosition.x + " " + headPosition.y + " " + headPosition.z);
66.         enable = true;
67.  
68.         listenForMessages();
69.     }
70.    
71.     // Update is called once per frame
72.     void Update ()
73.     {
74.         if (!enable) return;
75.  
76.         Debug.Log("Setting camera position: " + headPosition.x + " " + headPosition.y + " " + headPosition.z);
77.         this.transform.position = headPosition;
78.  
79.         Vector3 va = pa - headPosition;
80.         Vector3 vb = pb - headPosition;
81.         Vector3 vc = pc - headPosition;
82.  
83.         float d = -Vector3.Dot(vn, va);
84.  
85.         float l = Vector3.Dot(vr, va) * n / d;
86.         float r = Vector3.Dot(vr, vb) * n / d;
87.         float b = Vector3.Dot(vu, va) * n / d;
88.         float t = Vector3.Dot(vu, vc) * n / d;
89.  
90.         Matrix4x4 frustum = glFrustum(l, r, b, t, n, f);
91.         Matrix4x4 translate = glTranslatef(-headPosition.x, -headPosition.y, headPosition.z);
92.         Matrix4x4 projection = frustum * translate;
93.  
94.         camera.projectionMatrix = projection;
95.     }
96.    
97.     void LateUpdate()
98.     {
99.        
100.     }
101.  
102.     Matrix4x4 glFrustum(float l, float r, float b, float t, float n, float f)
103.     {
104.         Matrix4x4 m = new Matrix4x4();
105.         m[0, 0] = (2 * n) / (r - l);
106.         m[0, 1] = 0;
107.         m[0, 2] = (r + l) / (r - l);
108.         m[0, 3] = 0;
109.  
110.         m[1, 0] = 0;
111.         m[1, 1] = (2 * n) / (t - b);
112.         m[1, 2] = (t + b) / (t - b);
113.         m[1, 3] = 0;
114.  
115.         m[2, 0] = 0;
116.         m[2, 1] = 0;
117.         m[2, 2] = -(f + n) / (f - n);
118.         m[2, 3] = -(2 * f * n) / (f - n);
119.  
120.         m[3, 0] = 0;
121.         m[3, 1] = 0;
122.         m[3, 2] = -1;
123.         m[3, 3] = 0;
124.  
125.         return m;
126.     }
127.  
128.     Matrix4x4 glTranslatef(float x, float y, float z)
129.     {
130.         Matrix4x4 m = new Matrix4x4();
131.         m[0, 0] = 1;
132.         m[0, 1] = 0;
133.         m[0, 2] = 0;
134.         m[0, 3] = x;
135.  
136.         m[1, 0] = 0;
137.         m[1, 1] = 1;
138.         m[1, 2] = 0;
139.         m[1, 3] = y;
140.  
141.         m[2, 0] = 0;
142.         m[2, 1] = 0;
143.         m[2, 2] = 1;
144.         m[2, 3] = z;
145.  
146.         m[3, 0] = 0;
147.         m[3, 1] = 0;
148.         m[3, 2] = 0;
149.         m[3, 3] = 1;
150.         return m;
151.     }
152. }