One script taking 80% resources in Android

Hey everyone,

I have a mobile game in development that runs really well on iphone and on pc/mac but has a real problem on Android (HTC m8) or galaxy 4. low framerate 10fps where as pc is 300pfs and iphone 6 40+fps.

The script uses opencv but I have not found any informatiion that says it is real bad on android. The function that causes the issue is rects = _faces.ToArray(). Remove this and I get 40+ fps. Update function below. As you can see I have removed most of the function to try and isolate the problem, even with all the rest in it is only the call above that cause the issue.

Any help would be greatly appreciated.

Code (CSharp):

1.  
2. [LEFT]    void Update ()
3.     {
4.         // Make sure we've initialised.
5.         if ((_initDone) && (!_capturingFace)) {
6.             StartCoroutine ("CaptureFace");
7.         }
8.     }
9.    
10.     private IEnumerator CaptureFace() {
11.         _capturingFace = true;
12.        
13.         // Make sure that the texture has been correctly formated, if not we'll have to come back later.
14.         if (_webCamTexture.width > 16 && _webCamTexture.height > 16) {
15.            
16.             // Pass the web cam texture to a OpenCV matrix
17.             Utils.webCamTextureToMat (_webCamTexture, _rgbaMat, _colors);
18.            
19.             // iPhones buggering about with mirroring again...
20.             #if UNITY_IPHONE && !UNITY_EDITOR
21.             // Flip if neccessary
22.             if (_webCamTexture.videoVerticallyMirrored){
23.                 if(isFrontFacing){
24.                     Core.flip (_rgbaMat, _rgbaMat, 1);
25.                 }else{
26.                     Core.flip (_rgbaMat, _rgbaMat, 0);
27.                 }
28.             }else{
29.                 if(isFrontFacing){
30.                     Core.flip (_rgbaMat, _rgbaMat, -1);
31.                 }
32.             }
33.             #endif
34.            
35.             // Convert the rgb web texture matrix to gray
36.             Imgproc.cvtColor (_rgbaMat, _grayMat, Imgproc.COLOR_RGBA2GRAY);
37.            
38.             // Adjust the contrast - this can impact performance, try without for faster performance
39.             Imgproc.equalizeHist (_grayMat, _grayMat);
40.            
41.             // Set the cascade to detect different sized targets
42.             if (_cascadeFace != null)
43.                 _cascadeFace.detectMultiScale (_grayMat, _faces, 1.1, 2, 2, // TODO: objdetect.CV_HAAR_SCALE_IMAGE
44.                                                new Size (_webCamTexture.width * 0.15, _webCamTexture.width * 0.15), new Size ());
45.  
46.             //.Reset();
47.             sw.Start();
48.             Debug.Log("0 "+sw.ElapsedMilliseconds);
49.             AddToDebug ("0 "+sw.ElapsedMilliseconds);
50.             // Create an array of OpenCV rectangles from the array of faces.
51.             rects = _faces.toArray ();
52.             Debug.Log("1 "+sw.ElapsedMilliseconds);
53.             AddToDebug ("1 "+sw.ElapsedMilliseconds);
54.             // Find a mouth in each face each face.
55.             _faceFound = false;
56.  
57.             Debug.Log ("rects length "+rects.Length);
58. //            for (int i = 0; i < rects.Length; i++) {
59. //                Debug.Log ("rects length mk1"+rects.Length);
60. //                // We found a face
61. //                _faceFound = true;
62. //              
63. //                // For debugging, show us where the face is.
64. //                Core.rectangle (
65. //                    _rgbaMat,
66. //                    new Point (rects [i].x ,rects [i].y),
67. //                    new Point (rects [i].x + rects [i].width, rects [i].y + rects [i].height),
68. //                    new Scalar (0, 255, 0, 255),
69. //                    2);
70. //
71. //                Debug.Log("2 "+sw.ElapsedMilliseconds);
72. //                // Create a rectangle around the region of the face we're interested in
73. //                // Keep it inside the face box, otherwise we get errors when we're at the edge of the screen
74. //                /*                OpenCVForUnity.Rect _mouthROI = new OpenCVForUnity.Rect(
75. //                    (int)rects [i].x,
76. //                    (int)rects [i].y + (rects [i].height / 2),
77. //                    (int)rects [i].width,
78. //                    (int)(rects [i].height- (rects [i].height / 2)));
79. //*/                OpenCVForUnity.Rect _mouthROI = new OpenCVForUnity.Rect(            // IMPROVED SENSITIVITY!!!!
80. //                                                                             (int)rects [i].x,
81. //                                                                             (int)rects [i].y + ((rects [i].height / 3) * 2),
82. //                                                                             (int)rects [i].width,
83. //                                                                             (int)(rects [i].height - ((rects [i].height / 3) * 2)));
84. //              
85. //                // Create a new matrix using the ROI
86. //                _mouthMATGray = _grayMat.submat(_mouthROI);
87. //                _mouthMATRGBA = _rgbaMat.submat(_mouthROI);
88. //                Debug.Log("3 "+sw.ElapsedMilliseconds);
89. //                // Detect the mouth (we're only going to use the first mouth captured
90. //                // Set the cascade to detect different sized targets
91. //                if (_cascadeMouth != null)
92. //                    _cascadeMouth.detectMultiScale (_mouthMATGray, _mouths, 1.1, 2, 2, // TODO: objdetect.CV_HAAR_SCALE_IMAGE
93. //                                                    new Size (_webCamTexture.width * 0.04, _webCamTexture.width * 0.04), new Size ());
94. //                // Create an array of OpenCV rectangles from the array of mouths.
95. //                rectsM = _mouths.toArray ();
96. //              
97. //                // Put a rectangle around the first mouth on each face
98. //                _mouthFound = false;
99. ////                for (int j = 0; j < rectsM.Length; j++) {
100. ////                    // Set the mouth box and make sure that the x and y are correct (remember, these co-ords are inside the ROI)
101. ////                    _rectMouth = new UnityEngine.Rect(
102. ////                        rectsM[j].x + _mouthROI.x,
103. ////                        _webCamTexture.height - _mouthROI.y - rectsM[j].y - rectsM[j].height,
104. ////                        rectsM[j].width,
105. ////                        rectsM[j].height
106. ////                        );
107. ////                  
108. ////                    // We've found a mouth!
109. ////                    _mouthFound = true;
110. ////                    Debug.Log("4 "+sw.ElapsedMilliseconds);
111. ////                    Core.rectangle (
112. ////                        _rgbaMat,
113. ////                        new Point (rectsM [j].x + _mouthROI.x,rectsM [j].y + _mouthROI.y),
114. ////                        new Point (rectsM [j].x + rectsM [j].width + _mouthROI.x, rectsM [j].y + rectsM [j].height + _mouthROI.y),
115. ////                        new Scalar (0, 0, 255, 255),
116. ////                        2);
117. ////                    break;
118. ////                }
119. //            }
120.            
121.             // Add the output to the asssigned texture!
122.             Utils.matToTexture2D (_rgbaMat, _texture, _colors);
123.             Debug.Log("5 "+sw.ElapsedMilliseconds);
124.             // Are we debugging?
125.             _debugPreviewOn = _previewScript.isToggled;
126.             if (_debugPreviewOn) {
127.                 _webCamSprite = Sprite.Create(
128.                     _texture,
129.                     new UnityEngine.Rect(
130.                     0,
131.                     0,
132.                     _texture.width,
133.                     _texture.height),
134.                     new Vector2(0,0));
135.                 _previewImage.sprite = _webCamSprite;
136.             }
137.            
138.             //Debug.Log("Face:" + _faceFound.ToString() + " Mouth:" + _mouthFound.ToString());
139.            
140.            
141.            
142.             // Which version are we using??  The fact that we have a mouth with satisfy the "find/lose mouth" method
143.             if (logicSolution==1) {
144.                 // We're setting the oposite of mouth found (ie, if we've found the mouth it's closed
145.                 mouthIsOpenBoolean = !_mouthFound;
146.                 Debug.Log("6 "+sw.ElapsedMilliseconds);
147.             }
148.            
149.             // Only continue for solution 0 and we have a mouth.
150. //            if ((logicSolution==0) && (_mouthFound==true)) {
151. //                Debug.Log("We shouldn't see this");
152. //              
153. //                // How high and wide are the mouth
154. //                _mouthWidth = (int)_rectMouth.width;
155. //                _mouthHeight = (int)_rectMouth.height;
156. //              
157. //                // Create the mouth texture
158. //                Color[] pixels = _texture.GetPixels(
159. //                    (int)_rectMouth.x,
160. //                    (int)_rectMouth.y,
161. //                    (int)_rectMouth.width,
162. //                    (int)_rectMouth.height);
163. //                _textMouth.Resize(_mouthWidth, _mouthHeight);
164. //                _textMouth.Apply();
165. //                _textMouth.SetPixels(pixels);
166. //                _textMouth.Apply();
167. //              
168. //                // Work out detection pixels
169. //                int mouthCentre = _mouthWidth / 2;
170. //                _totalBright = 0;
171. //                int mouthGap = _mouthHeight / mouthDetectPixelCount;
172. //                UnityEngine.Rect rectShapePreview =
173. //                    new UnityEngine.Rect((float)mouthCentre - 1, (float)mouthGap,3, (float)mouthGap*(float)mouthDetectPixelCount);
174. //              
175. //                // Work out the gap between the pixels
176. //                for (int i = 1; i <= mouthDetectPixelCount; i++)
177. //                {
178. //                    // Work out the total brightness
179. //                    int pixelY = mouthGap * i;
180. //                    _totalBright += Brightness(_textMouth.GetPixel(mouthCentre, pixelY));
181. //                }
182.                
183.                 // Have we opened or closed.
184.                 //
185.                 // Rules will be (for boolean):
186.                 //    1. The initial change will be 2 tenths in eather direction (double mouthChangeDiff) to change open or closed.
187.                 //        Smaller = Open, Bigger = Closed.
188.                 //    2. Record the current number into _lastBright (double) and start closed (bool - false) in mouthIsOpenBoolean
189.                 //    3. If the number gets bigger keep updating the _lastBright variable to make note of the "close" brightness.
190.                 //    4. If the number gets smaller measure it, if it's more then 2/10s (or what ever the variable is we're now open:
191.                 //        change the bool to true and record the number.
192.                 //        If it's less just leave it, this will help us if we capture the mouth when it's mid open or mid close.
193.                 //    5. If we've moved to closed do stops to to 4 but in the oposite direction
194.                 //
195.                 // Rules will be (for integer)
196.                 //     1. Use boolean logic but the change diff maximimum is 100 (open - oposite to boolean :S) and minimum is 0 (closed);
197.                 //    2. Use the increment to tie the setting to the nearest increment.
198.                
199.                 // Record the current number into _lastBright if this is the first capture
200. //                if (_lastBright == 0) {
201. //                    _lastBright = _totalBright;
202. ////                }
203. //                else {
204. //                    //Get the difference
205. //                    double diff = new double();
206. //                    diff = _lastBright - _totalBright;
207. //                  
208. //                    // We need a positive diff too
209. //                    double posDiff = diff;
210. //                    if (posDiff < 0) {posDiff = -posDiff;}
211. //                  
212. //                    // Have we gone more open or more closed?
213. //                    if (((_lastBright > _totalBright) && (mouthIsOpenBoolean)) ||
214. //                        ((_lastBright < _totalBright) && (!mouthIsOpenBoolean))) {
215. //                      
216. //                        // We're going further in the same direction, log the last brighness
217. //                        _lastBright = _totalBright;
218. //                    }
219. //                    else {
220. //                        // We've changed direction.  If it is more than the diff level set???
221. //                        if (posDiff > mouthChangeDiff) {
222. //                            // We've changed direction!
223. //                            if (mouthIsOpenBoolean) {
224. //                                mouthIsOpenBoolean = false;
225. //                            }
226. //                            else {
227. //                                mouthIsOpenBoolean = true;
228. //                            }
229. //                          
230. //                            // Set the new last brightness
231. //                            _lastBright = _totalBright;
232. //                        }  
233. //                    }
234.                 //}
235.             //}
236.         }
237.        
238.         yield return new WaitForSeconds (0.1f);        // Massive performance boost.
239.         _capturingFace = false;
240.         yield return null;[/LEFT]
241.     }

 

Sorry to bump but hoping for a experienced c# coder to rescue my sanity. There should be no reason this runs so poorly on Android but fine on IOS that I can find.

Anyone enlighten me?

 

What's the datatype of _faces?

 

Andy, you might have more luck asking on the OpenCV forums since, as you've already determined, it is the call to that specific library which performs slowly.

Without better knowledge of the ToArray method, I'm afraid we won't have much chance of helping.

To be clear, I'm not suggesting you shouldn't have asked here. I just know, for myself at least, I skipped over this thread because I don't have enough information to even attempt helping. Just thought the folks on at OpenCV probably have more experience with this.

 

Thanks for the reply, Even though you have made me feel a bit silly for not thinking to ask there myself he he.


Very good shout and much appreciated.

 

private MatOfRect _faces;

_faces is a matrix datatype

 

Do you absolutely need to use toArray? I'm looking at the source for opencvsharp, I don't know if this is the wrapper you're using, but at least there is both an indexer and iterator you could try using instead.

It's hard to say why the toArray method would be so expensive, you'd have to profile deeper in the wrapper and the native code.

 

Yeah maybe iterating through it might be less expensive. There are two of us in the project and my partner is mainly working with opencv so I will pass it on.

He did say there is not a large array and sometimes only one face so it might make sense not to use toarray.

We use the opencvforunity asset and a matrix of faces to compare. He thinks that it might be todo with the higher res camera's on android compared to his iphone so we are going to try and limit that for the multiscale part of the code.

Thanks for taking the time to have a look, it is a strange one with it running so well on non android.