[unity windows] build a kinect plugin : freeze on call to NuiImageStreamGetNextFrame

Hi,

I m workin on a kinect plugin for unity 3D 3.5 and I m stuck with the video data

I successfully init the kinect like this and all is ok with skeleton data

Code (csharp):

1.  
2. bool KinectManager::InitNuiSensor()
3. {
4.     INuiSensor * pNuiSensor;
5.  
6.     status = 0;
7.  
8.     int iSensorCount = 0;
9.     HRESULT hr = NuiGetSensorCount(&iSensorCount);
10.     if (FAILED(hr))
11.     {
12.         return false;
13.     }
14.  
15.     // Look at each Kinect sensor
16.     for (int i = 0; i < iSensorCount; ++i)
17.     {
18.         // Create the sensor so we can check status, if we can't create it, move on to the next
19.         hr = NuiCreateSensorByIndex(i, &pNuiSensor);
20.         if (FAILED(hr))
21.         {
22.             continue;
23.         }
24.  
25.         // Get the status of the sensor, and if connected, then we can initialize it
26.         hr = pNuiSensor->NuiStatus();
27.         if (S_OK == hr)
28.         {
29.             m_pNuiSensor = pNuiSensor;
30.             break;
31.         }
32.  
33.         // This sensor wasn't OK, so release it since we're not using it
34.         pNuiSensor->Release();
35.     }
36.  
37.     if (NULL != m_pNuiSensor)
38.     {
39.         // Initialize the Kinect and specify that we'll be using skeleton
40.         hr = m_pNuiSensor->NuiInitialize(NUI_INITIALIZE_FLAG_USES_SKELETON | NUI_INITIALIZE_FLAG_USES_COLOR);
41.  
42.  
43.         if (SUCCEEDED(hr))
44.         {
45.             // Create an event that will be signaled when skeleton data is available
46.             m_hNextSkeletonEvent = CreateEventW(NULL, TRUE, FALSE, NULL);
47.  
48.             // Open a skeleton stream to receive skeleton data
49.             hr = m_pNuiSensor->NuiSkeletonTrackingEnable(m_hNextSkeletonEvent, 0);
50.  
51.  
52.            
53.             nextColorFrameEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
54.  
55.             hr = m_pNuiSensor->NuiImageStreamOpen(NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480, 0, 2, nextColorFrameEvent, &videoStreamHandle);
56.  
57.             if (FAILED(hr))
58.             {
59.                   return false;
60.             }
61.         }
62.     }
63.  
64.     if (NULL == m_pNuiSensor || FAILED(hr))
65.     {
66.       return false;
67.     }
68.  
69.     status = 1;
70.  
71.     return true;
72.  
73. }
74.  

I want to get a video frame and upload it as a texture in unity

In order to get this done, I ve build a GetPicture method

Code (csharp):

1.  
2. long KinectManager::GetPicture(unsigned char **buffer)
3. {
4.     HRESULT hr;
5.  
6.     NUI_IMAGE_FRAME imageFrame;
7.  
8.      // Attempt to get the color frame
9.         hr = m_pNuiSensor->NuiImageStreamGetNextFrame(nextColorFrameEvent, 0, &imageFrame);
10.  
11.      
12.     if( FAILED( hr ) )
13.     {
14.              return hr;
15.     }
16.  
17.     INuiFrameTexture * pTexture = imageFrame.pFrameTexture;
18.  
19.     NUI_LOCKED_RECT LockedRect;
20.  
21.     pTexture->LockRect( 0, &LockedRect, NULL, 0 );
22.  
23.     if( LockedRect.Pitch != 0 )
24.     {
25.  
26.             unsigned char *pBuffer = (unsigned char*) LockedRect.pBits;
27.  
28.             NUI_IMAGE_RESOLUTION resolution = imageFrame.eResolution;
29.        
30.             if(resolution == NUI_IMAGE_RESOLUTION_640x480)
31.             {
32.                 *buffer = pBuffer;
33.             }
34.       }
35.  
36.  
37.  
38.       return S_OK;
39. }
40.  
41.  

the problem is that unity freeze on the call to NuiImageStreamGetNextFrame.

If I remove the line nextColorFrameEvent = CreateEvent(NULL, TRUE, FALSE, NULL); in init. I dont get the freeze any more, but buffer remains empty.

Any suggestions?

 

I m stuck with this.. any ideas guys ? thanks !

 

I finally got it to work !

I was missing the UnlockRect(0) line of code on the C++ size which was the root cause of the freeze

anyway, now I ve got problems on the unity size.

I get a out of memory error after about a minute of run.

looks like when I replace image in texture2d I get a big leak.

How to tell unity to delete the previous instance of the texture

Code (csharp):

1.  
2. IntPtr  ptrImageBuffer = IntPtr.Zero;
3.  
4.                 int result = GetPicture(ref ptrImageBuffer);
5.    
6.                 if (ptrImageBuffer != IntPtr.Zero )
7.                 {
8.                    
9.                     if (this.textCam != null)
10.                     {
11.                         this.textCam =null;
12.                     }
13.                    
14.                     this.textCam = new Texture2D(640,480);
15.                    
16.                     for (int x=0;x<640;x++)
17.                     {
18.                         for (int y=0;y<480;y++)
19.                         {
20.                            
21.                             int posbrush = x*4 + ((480-y) * 640 *4);
22.                    
23.                             int val = Marshal.ReadInt32(ptrImageBuffer, posbrush);
24.        
25.                             int red   = (val  0x00ff0000) >> 16;
26.                             int green = (val  0x0000ff00) >> 8;
27.                             int blue  = (val  0x000000ff);
28.        
29.                             Color c = new Color( (float)(red / 255.0f),(float)(green / 255.0f), (float)(blue  / 255.0f) ,1.0f);
30.                            
31.                             textCam.SetPixel(x,y,c);
32.                            
33.                            
34.                         }
35.                        
36.                     }
37.                    
38.                     textCam.Apply();
39.  
40.