Can I add more pointers to an InputModule?

I'm writing a custom InputModule that's supposed to handle VR controllers, gaze based "controllers" and mouse input. They interact with a UI rendered to a rendertexture on a sphere. All raycasting works properly. I only have an issue getting the data into the EventSystem properly.

It seems that out of the box an InputModule only handles 1 pointer. This means I can only hover a single UI element at a time. While trying to find a way around this, I noticed that pointers are coupled with an id, so I tried assigning custom ids to my PointerEventData (one for each controller), but now I'm getting NullReferenceExceptions in the base StandaloneInputModule:

Code (csharp):

1. NullReferenceException: Object reference not set to an instance of an object
2. UnityEngine.EventSystems.StandaloneInputModule.ProcessMousePress (UnityEngine.EventSystems.PointerInputModule+MouseButtonEventData data) (at C:/buildslave/unity/build/Extensions/guisystem/UnityEngine.UI/EventSystem/InputModules/StandaloneInputModule.cs:499)
3. UnityEngine.EventSystems.StandaloneInputModule.ProcessMouseEvent (System.Int32 id) (at C:/buildslave/unity/build/Extensions/guisystem/UnityEngine.UI/EventSystem/InputModules/StandaloneInputModule.cs:471)
4. UnityEngine.EventSystems.StandaloneInputModule.ProcessMouseEvent () (at C:/buildslave/unity/build/Extensions/guisystem/UnityEngine.UI/EventSystem/InputModules/StandaloneInputModule.cs:446)
5. UnityEngine.EventSystems.StandaloneInputModule.Process () (at C:/buildslave/unity/build/Extensions/guisystem/UnityEngine.UI/EventSystem/InputModules/StandaloneInputModule.cs:221)
6. UnityEngine.EventSystems.EventSystem.Update () (at C:/buildslave/unity/build/Extensions/guisystem/UnityEngine.UI/EventSystem/EventSystem.cs:294)
7.  

Which seems to indicate that the Pointer system doesn't work as I thought. Do they only work with the predefined ids? (i.e. int kMouseLeftId = -1, int kMouseRightId = -2, int kMouseMiddleId = -3, kFakeTouchesId = -4)

My InputModule for reference.

Code (CSharp):

1. using System.Collections.Generic;
2. using UnityEngine;
3. using UnityEngine.EventSystems;
4. using UnityEngine.XR;
5.  
6. public class SphereUIInputModule: StandaloneInputModule
7. {
8.     private new Camera camera;
9.     private RenderTexture uiTexture;
10.  
11.     private readonly MouseState mouseState = new MouseState();
12.     //Since we can have multiple input devices at the same time, we store all intermediate results in lists.
13.     private Dictionary<int, PointerEventData> pointers;
14.     private Dictionary<int, Vector3> directions;
15.     private Dictionary<int, Vector2> positions;
16.     private Dictionary<int, Vector2> positionResults;
17.     private Dictionary<int, RaycastResult> raycastResults;
18.     private Dictionary<int, PointerEventData.FramePressState> clickStates;
19.  
20.     public Controller leftController;
21.     public Controller rightController;
22.  
23.     //My custom pointer ids. They extend the default ids defined in PointerInputModule:
24.     // public const int kMouseLeftId = -1;
25.     // public const int kMouseRightId = -2;
26.     // public const int kMouseMiddleId = -3;
27.     // public const int kFakeTouchesId = -4;
28.     private const int gazeId = 1;
29.     private const int rightControllerId = 2;
30.     private const int leftControllerId = 3;
31.  
32.  
33.     protected override void Awake()
34.     {
35.         pointers = new Dictionary<int, PointerEventData>();
36.         directions = new Dictionary<int, Vector3>();
37.         positions = new Dictionary<int, Vector2>();
38.         positionResults = new Dictionary<int, Vector2>();
39.         raycastResults = new Dictionary<int, RaycastResult>();
40.         clickStates = new Dictionary<int, PointerEventData.FramePressState>();
41.  
42.         camera = Camera.main;
43.         uiTexture = GetComponent<Camera>().targetTexture;
44.         base.Awake();
45.     }
46.  
47.     //Figure out whether a VR controller trigger has been pressed or released this frame
48.     protected PointerEventData.FramePressState StateForControllerTrigger(Controller controller)
49.     {
50.         var pressed = controller.triggerPressed;
51.         var released = controller.triggerReleased;
52.  
53.         if (pressed && released)
54.         {
55.             return PointerEventData.FramePressState.PressedAndReleased;
56.         }
57.         if (pressed)
58.         {
59.             return PointerEventData.FramePressState.Pressed;
60.         }
61.         if (released)
62.         {
63.             return PointerEventData.FramePressState.Released;
64.         }
65.  
66.         return PointerEventData.FramePressState.NotChanged;
67.     }
68.  
69.    
70.     //Since we can have multiple pointers active at the same time (2 controllers for example), we need to store all intermediate results in lists.
71.     protected override MouseState GetMousePointerEventData(int id)
72.     {
73.         PointerEventData leftData;
74.         GetPointerData(kMouseLeftId, out leftData, true);
75.        
76.         leftData.Reset();
77.  
78.         //Figure out which input devices are active, and get a Ray for each one.
79.         directions.Clear();
80.         if (XRSettings.enabled)
81.         {
82.             if (VRDevices.loadedControllerSet == VRDevices.LoadedControllerSet.NoControllers)
83.             {
84.                 directions.Add(gazeId, camera.ScreenPointToRay(new Vector2(Screen.width, Screen.height)).direction);
85.             }
86.             else
87.             {
88.                 if (VRDevices.hasRightController)
89.                 {
90.                     directions.Add(leftControllerId, rightController.GetComponent<Controller>().CastRay().direction);
91.                 }
92.                 if (VRDevices.hasLeftController)
93.                 {
94.                     directions.Add(rightControllerId, leftController.GetComponent<Controller>().CastRay().direction);
95.                 }
96.             }
97.         }
98.         if (Input.mousePresent)
99.         {
100.             directions.Add(kMouseLeftId, camera.ScreenPointToRay((Vector2)Input.mousePosition).direction);
101.         }
102.  
103.         //Convert each ray from spherical coordinates to canvas coordinates
104.         positions.Clear();
105.         foreach (var direction in directions)
106.         {
107.             positions.Add(direction.Key, new Vector2
108.             {
109.                 x = uiTexture.width * (0.5f - Mathf.Atan2(direction.Value.z, direction.Value.x) / (2f * Mathf.PI)),
110.                 y = uiTexture.height * (Mathf.Asin(direction.Value.y) / Mathf.PI + 0.5f)
111.             });
112.         }
113.  
114.         //Figure out which UI elements are being hit by the raycasts
115.         raycastResults.Clear();
116.         positionResults.Clear();
117.         foreach (var position in positions)
118.         {
119.             var tempData = new PointerEventData(eventSystem);
120.             tempData.Reset();
121.  
122.             tempData.position = position.Value;
123.  
124.             eventSystem.RaycastAll(tempData, m_RaycastResultCache);
125.             var result = FindFirstRaycast(m_RaycastResultCache);
126.             if (result.isValid)
127.             {
128.                 raycastResults.Add(position.Key, result);
129.                 positionResults.Add(position.Key, position.Value);
130.             }
131.             m_RaycastResultCache.Clear();
132.         }
133.  
134.         //Store all relevant data for each input device into a PointerEventData
135.         pointers.Clear();
136.         foreach (var kvp in raycastResults)
137.         {
138.             PointerEventData prevData;
139.             GetPointerData(kvp.Key, out prevData, true);
140.  
141.             pointers.Add(kvp.Key, new PointerEventData(eventSystem)
142.             {
143.                 delta = positionResults[kvp.Key] - prevData.position,
144.                 position = positionResults[kvp.Key],
145.                 scrollDelta = Input.mouseScrollDelta,
146.                 button = PointerEventData.InputButton.Left,
147.                 pointerCurrentRaycast = raycastResults[kvp.Key],
148.                 pointerId = kvp.Key,
149.             });
150.         }
151.  
152.         //Figure out whether each input device has pressed or released this frame.
153.         clickStates.Clear();
154.         clickStates.Add(leftControllerId, StateForControllerTrigger(leftController));
155.         clickStates.Add(rightControllerId, StateForControllerTrigger(leftController));
156.         clickStates.Add(kMouseLeftId, StateForMouseButton(0));
157.  
158.         //Store all the pointers in a mouseState.
159.         foreach (var kvp in pointers)
160.         {
161.             mouseState.SetButtonState(PointerEventData.InputButton.Left, clickStates[kvp.Key], pointers[kvp.Key]);
162.         }
163.  
164.         return mouseState;
165.     }
166. }

 

I've found somewhat of a solution to using multiple pointers. Unfortunately I now have to handle sending Events myself:

Code (CSharp):

1. using System.Collections.Generic;
2. using UnityEngine;
3. using UnityEngine.EventSystems;
4. using UnityEngine.XR;
5.  
6. public class SphereUIInputModule: StandaloneInputModule
7. {
8.     private new Camera camera;
9.     private RenderTexture uiTexture;
10.  
11.     private Dictionary<int, PointerEventData> pointers;
12.     private Dictionary<int, Vector3> directions;
13.     private Dictionary<int, Vector2> positions;
14.     private Dictionary<int, Vector2> positionResults;
15.     private Dictionary<int, RaycastResult> raycastResults;
16.     private Dictionary<int, PointerEventData.FramePressState> clickStates;
17.     private Dictionary<int, GameObject> previousHovers;
18.  
19.     public Controller leftController;
20.     public Controller rightController;
21.  
22.     public RectTransform[] DebugPointers;
23.  
24.     private const int gazeId = 1;
25.     private const int rightControllerId = 2;
26.     private const int leftControllerId = 3;
27.  
28.  
29.     protected override void Awake()
30.     {
31.         pointers = new Dictionary<int, PointerEventData>();
32.         directions = new Dictionary<int, Vector3>();
33.         positions = new Dictionary<int, Vector2>();
34.         positionResults = new Dictionary<int, Vector2>();
35.         raycastResults = new Dictionary<int, RaycastResult>();
36.         clickStates = new Dictionary<int, PointerEventData.FramePressState>();
37.         previousHovers = new Dictionary<int, GameObject>();
38.  
39.         camera = Camera.main;
40.         uiTexture = GetComponent<Camera>().targetTexture;
41.         base.Awake();
42.     }
43.  
44.     protected PointerEventData.FramePressState StateForControllerTrigger(Controller controller)
45.     {
46.         var pressed = controller.triggerPressed;
47.         var released = controller.triggerReleased;
48.  
49.         if (pressed && released)
50.         {
51.             return PointerEventData.FramePressState.PressedAndReleased;
52.         }
53.         if (pressed)
54.         {
55.             return PointerEventData.FramePressState.Pressed;
56.         }
57.         if (released)
58.         {
59.             return PointerEventData.FramePressState.Released;
60.         }
61.  
62.         return PointerEventData.FramePressState.NotChanged;
63.     }
64.  
65.     public override void Process()
66.     {
67.         PointerEventData leftData;
68.         GetPointerData(kMouseLeftId, out leftData, true);
69.        
70.         leftData.Reset();
71.  
72.         //NOTE(Simon): There could be more than 1 inputdevice (VR controllers for example), so store them all in a list
73.         directions.Clear();
74.         if (XRSettings.enabled)
75.         {
76.             if (VRDevices.loadedControllerSet == VRDevices.LoadedControllerSet.NoControllers)
77.             {
78.                 directions.Add(gazeId, camera.ScreenPointToRay(new Vector2(Screen.width, Screen.height)).direction);
79.             }
80.             else
81.             {
82.                 if (VRDevices.hasRightController)
83.                 {
84.                     directions.Add(rightControllerId, rightController.GetComponent<Controller>().CastRay().direction);
85.                 }
86.                 if (VRDevices.hasLeftController)
87.                 {
88.                     directions.Add(leftControllerId, leftController.GetComponent<Controller>().CastRay().direction);
89.                 }
90.             }
91.         }
92.         if (Input.mousePresent)
93.         {
94.             directions.Add(kMouseLeftId, camera.ScreenPointToRay((Vector2)Input.mousePosition).direction);
95.         }
96.  
97.         positions.Clear();
98.         foreach (var direction in directions)
99.         {
100.             positions.Add(direction.Key, new Vector2
101.             {
102.                 x = uiTexture.width * (0.5f - Mathf.Atan2(direction.Value.z, direction.Value.x) / (2f * Mathf.PI)),
103.                 y = uiTexture.height * (Mathf.Asin(direction.Value.y) / Mathf.PI + 0.5f)
104.             });
105.         }
106.  
107.         var debugPositions = new List<Vector2>(positions.Values);
108.         DrawDebugPointers(debugPositions);
109.  
110.         raycastResults.Clear();
111.         positionResults.Clear();
112.         foreach (var position in positions)
113.         {
114.             var tempData = new PointerEventData(eventSystem);
115.             tempData.Reset();
116.  
117.             tempData.position = position.Value;
118.  
119.             eventSystem.RaycastAll(tempData, m_RaycastResultCache);
120.             var result = FindFirstRaycast(m_RaycastResultCache);
121.             if (result.isValid)
122.             {
123.                 raycastResults.Add(position.Key, result);
124.                 positionResults.Add(position.Key, position.Value);
125.             }
126.             m_RaycastResultCache.Clear();
127.         }
128.  
129.         pointers.Clear();
130.         foreach (var kvp in raycastResults)
131.         {
132.             PointerEventData prevData;
133.             GetPointerData(kvp.Key, out prevData, true);
134.  
135.             pointers.Add(kvp.Key, new PointerEventData(eventSystem)
136.             {
137.                 delta = positionResults[kvp.Key] - prevData.position,
138.                 position = positionResults[kvp.Key],
139.                 scrollDelta = Input.mouseScrollDelta,
140.                 button = PointerEventData.InputButton.Left,
141.                 pointerCurrentRaycast = raycastResults[kvp.Key],
142.                 pointerId = kvp.Key,
143.             });
144.         }
145.  
146.         //TODO(Simon): Add hover clickstate determination
147.         clickStates.Clear();
148.         clickStates.Add(leftControllerId, StateForControllerTrigger(leftController));
149.         clickStates.Add(rightControllerId, StateForControllerTrigger(rightController));
150.         clickStates.Add(kMouseLeftId, StateForMouseButton(0));
151.  
152.         foreach (var kvp in pointers)
153.         {
154.             if (!previousHovers.ContainsKey(kvp.Key))
155.             {
156.                 previousHovers.Add(kvp.Key, null);
157.             }
158.             if (kvp.Value.pointerCurrentRaycast.gameObject != previousHovers[kvp.Key])
159.             {
160.                 ExecuteEvents.ExecuteHierarchy(kvp.Value.pointerCurrentRaycast.gameObject, kvp.Value, ExecuteEvents.pointerEnterHandler);
161.                 //NOTE(Simon): Check if any other pointers aree hovering the object that's just been unhovered.
162.                 var otherHovers = false;
163.                 foreach (var currentHover in pointers)
164.                 {
165.                     if (currentHover.Value.pointerCurrentRaycast.gameObject == previousHovers[kvp.Key])
166.                     {
167.                         otherHovers = true;
168.                     }
169.                 }
170.                 //NOTE(Simon): If no other hovers, send PointerExit Event.
171.                 if (!otherHovers)
172.                 {
173.                     ExecuteEvents.ExecuteHierarchy(previousHovers[kvp.Key], kvp.Value, ExecuteEvents.pointerExitHandler);
174.                 }
175.                 previousHovers[kvp.Key] = kvp.Value.pointerCurrentRaycast.gameObject;
176.             }
177.             if (clickStates[kvp.Key] == PointerEventData.FramePressState.Pressed || clickStates[kvp.Key] == PointerEventData.FramePressState.PressedAndReleased)
178.             {
179.                 ExecuteEvents.ExecuteHierarchy(kvp.Value.pointerCurrentRaycast.gameObject, kvp.Value, ExecuteEvents.pointerClickHandler);
180.             }
181.         }
182.     }
183.  
184.     public void DrawDebugPointers(List<Vector2> locations)
185.     {
186.         for (int i = 0; i < locations.Count; i++)
187.         {
188.             DebugPointers[i].anchoredPosition = locations[i];
189.         }
190.     }
191. }

 

Any update on this? I am also trying to use 2 pointers within 1 EventSystem for VR purposes.

 

Would also like to see a simple solution to this!