[RELEASED] OpenCV for Unity

http://forum.unity3d.com/threads/released-opencv-for-unity.277080/page-16#post-2709724
Did you failed to generate the video file when you test this sample?

 

Since opencv2.framework has been added bitcode, framework size become large.
http://forum.unity3d.com/threads/unity-5-3-x-build-size-increase-faq.383533/

for now ,I do not plan to publish smaller size version(reduce support modules).

 

Hello
please can you reply at least?

 

i need to get the center of the rectangle on the camshift scene , it seems that isn't the same that simple face detection scene.....i nee your help

 

Sorry for my late reply.

Code (CSharp):

1. using UnityEngine;
2. using System.Collections;
3.  
4. #if UNITY_5_3 || UNITY_5_3_OR_NEWER
5. using UnityEngine.SceneManagement;
6. #endif
7. using OpenCVForUnity;
8.  
9. using System.Collections.Generic;
10.  
11. namespace OpenCVForUnitySample
12. {
13.         /// <summary>
14.         /// CamShift sample.
15.         /// referring to the http://www.computervisiononline.com/blog/tutorial-using-camshift-track-objects-video.
16.         /// </summary>
17.         [RequireComponent(typeof(WebCamTextureToMatHelper))]
18.         public class CamShiftSample : MonoBehaviour
19.         {
20.  
21.                 /// <summary>
22.                 /// The colors.
23.                 /// </summary>
24.                 Color32[] colors;
25.  
26.                 /// <summary>
27.                 /// The texture.
28.                 /// </summary>
29.                 Texture2D texture;
30.  
31.                 /// <summary>
32.                 /// The roi point list.
33.                 /// </summary>
34.                 List<Point> roiPointList;
35.  
36.                 /// <summary>
37.                 /// The roi rect.
38.                 /// </summary>
39.                 OpenCVForUnity.Rect roiRect;
40.  
41.                 /// <summary>
42.                 /// The hsv mat.
43.                 /// </summary>
44.                 Mat hsvMat;
45.  
46.                 /// <summary>
47.                 /// The roi hist mat.
48.                 /// </summary>
49.                 Mat roiHistMat;
50.  
51.                 /// <summary>
52.                 /// The termination.
53.                 /// </summary>
54.                 TermCriteria termination;
55.  
56.                 /// <summary>
57.                 /// The web cam texture to mat helper.
58.                 /// </summary>
59.                 WebCamTextureToMatHelper webCamTextureToMatHelper;
60.  
61.  
62.  
63.                 public GameObject point3D;
64.  
65.  
66.    
67.                 // Use this for initialization
68.                 void Start ()
69.                 {
70.                         roiPointList = new List<Point> ();
71.                         termination = new TermCriteria (TermCriteria.EPS | TermCriteria.COUNT, 10, 1);
72.                        
73.                         webCamTextureToMatHelper = gameObject.GetComponent<WebCamTextureToMatHelper> ();
74.                         webCamTextureToMatHelper.Init ();
75.  
76.                 }
77.  
78.                 /// <summary>
79.                 /// Raises the web cam texture to mat helper inited event.
80.                 /// </summary>
81.                 public void OnWebCamTextureToMatHelperInited ()
82.                 {
83.                         Debug.Log ("OnWebCamTextureToMatHelperInited");
84.            
85.                         Mat webCamTextureMat = webCamTextureToMatHelper.GetMat ();
86.            
87.                         colors = new Color32[webCamTextureMat.cols () * webCamTextureMat.rows ()];
88.                         texture = new Texture2D (webCamTextureMat.cols (), webCamTextureMat.rows (), TextureFormat.RGBA32, false);
89.  
90.                         hsvMat = new Mat (webCamTextureMat.rows (), webCamTextureMat.cols (), CvType.CV_8UC3);
91.            
92.                         gameObject.transform.localScale = new Vector3 (webCamTextureMat.cols (), webCamTextureMat.rows (), 1);
93.            
94.                         Debug.Log ("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
95.            
96.                         float width = 0;
97.                         float height = 0;
98.            
99.                         width = gameObject.transform.localScale.x;
100.                         height = gameObject.transform.localScale.y;
101.            
102.                         float widthScale = (float)Screen.width / width;
103.                         float heightScale = (float)Screen.height / height;
104.                         if (widthScale < heightScale) {
105.                                 Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
106.                         } else {
107.                                 Camera.main.orthographicSize = height / 2;
108.                         }
109.            
110.                         gameObject.GetComponent<Renderer> ().material.mainTexture = texture;
111.  
112.                 }
113.  
114.                 /// <summary>
115.                 /// Raises the web cam texture to mat helper disposed event.
116.                 /// </summary>
117.                 public void OnWebCamTextureToMatHelperDisposed ()
118.                 {
119.                         Debug.Log ("OnWebCamTextureToMatHelperDisposed");
120.  
121.                         hsvMat.Dispose ();
122.                         if (roiHistMat != null)
123.                                 roiHistMat.Dispose ();
124.                         roiPointList.Clear ();
125.                 }
126.  
127.                 // Update is called once per frame
128.                 void Update ()
129.                 {
130.  
131.                         if (webCamTextureToMatHelper.isPlaying () && webCamTextureToMatHelper.didUpdateThisFrame ()) {
132.                
133.                                 Mat rgbaMat = webCamTextureToMatHelper.GetMat ();
134.  
135.                                 Imgproc.cvtColor (rgbaMat, hsvMat, Imgproc.COLOR_RGBA2RGB);
136.                                 Imgproc.cvtColor (hsvMat, hsvMat, Imgproc.COLOR_RGB2HSV);
137.                
138.                
139.                                 Point[] points = roiPointList.ToArray ();
140.                
141.                                 if (roiPointList.Count == 4) {
142.                                                        
143.                
144.                                         using (Mat backProj = new Mat ()) {
145.                                                 Imgproc.calcBackProject (new List<Mat> (new Mat[]{hsvMat}), new MatOfInt (0), roiHistMat, backProj, new MatOfFloat (0, 180), 1.0);
146.                
147.                                                 RotatedRect r = Video.CamShift (backProj, roiRect, termination);
148.                                                 r.points (points);
149.                                         }
150.                
151.                                         #if ((UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR)
152.                                             //Touch
153.                                             int touchCount = Input.touchCount;
154.                                             if (touchCount == 1)
155.                                             {
156.                                                
157.                                                 if(Input.GetTouch(0).phase == TouchPhase.Ended){
158.                                                    
159.                                                     roiPointList.Clear ();
160.                                                 }
161.                                                
162.                                             }
163.                                         #else
164.                                         if (Input.GetMouseButtonUp (0)) {
165.                                                 roiPointList.Clear ();
166.                                         }
167.                                         #endif
168.                                 }
169.                
170.                
171.                                 if (roiPointList.Count < 4) {
172.                
173.                                         #if ((UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR)
174.                                             //Touch
175.                                             int touchCount = Input.touchCount;
176.                                             if (touchCount == 1)
177.                                             {
178.                                                 Touch t = Input.GetTouch(0);
179.                                                 if(t.phase == TouchPhase.Ended){
180.                                                     roiPointList.Add (convertScreenPoint (new Point (t.position.x, t.position.y), gameObject, Camera.main));
181.                 //                                    Debug.Log ("touch X " + t.position.x);
182.                 //                                    Debug.Log ("touch Y " + t.position.y);
183.                
184.                                                     if (!(new OpenCVForUnity.Rect (0, 0, hsvMat.width (), hsvMat.height ()).contains (roiPointList [roiPointList.Count - 1]))) {
185.                                                         roiPointList.RemoveAt (roiPointList.Count - 1);
186.                                                     }
187.                                                 }
188.                                                
189.                                             }
190.                                         #else
191.                                         //Mouse
192.                                         if (Input.GetMouseButtonUp (0)) {
193.                                                                
194.                                                 roiPointList.Add (convertScreenPoint (new Point (Input.mousePosition.x, Input.mousePosition.y), gameObject, Camera.main));
195.                                                 //                                                Debug.Log ("mouse X " + Input.mousePosition.x);
196.                                                 //                                                Debug.Log ("mouse Y " + Input.mousePosition.y);
197.                
198.                                                 if (!(new OpenCVForUnity.Rect (0, 0, hsvMat.width (), hsvMat.height ()).contains (roiPointList [roiPointList.Count - 1]))) {
199.                                                         roiPointList.RemoveAt (roiPointList.Count - 1);
200.                                                 }
201.                                         }
202.                                         #endif
203.                
204.                                            
205.                                         if (roiPointList.Count == 4) {
206.                
207.                                                 using (MatOfPoint roiPointMat = new MatOfPoint (roiPointList.ToArray ())) {
208.                                                         roiRect = Imgproc.boundingRect (roiPointMat);
209.                                                 }
210.                
211.                                                
212.                                                 if (roiHistMat != null) {
213.                                                         roiHistMat.Dispose ();
214.                                                         roiHistMat = null;
215.                                                 }
216.                                                 roiHistMat = new Mat ();
217.                
218.                                                 using (Mat roiHSVMat = new Mat(hsvMat, roiRect))
219.                                                 using (Mat maskMat = new Mat ()) {
220.                
221.                                                                        
222.                                                         Imgproc.calcHist (new List<Mat> (new Mat[]{roiHSVMat}), new MatOfInt (0), maskMat, roiHistMat, new MatOfInt (16), new MatOfFloat (0, 180));
223.                                                         Core.normalize (roiHistMat, roiHistMat, 0, 255, Core.NORM_MINMAX);
224.                                                
225.                                                         //                                                        Debug.Log ("roiHist " + roiHistMat.ToString ());
226.                                                 }
227.                                         }
228.                                 }
229.                
230.                                 if (points.Length < 4) {
231.                
232.                                         for (int i = 0; i < points.Length; i++) {
233.                                                 Imgproc.circle (rgbaMat, points [i], 6, new Scalar (0, 0, 255, 255), 2);
234.                                         }
235.                
236.                                 } else {
237.                
238.                                         for (int i = 0; i < 4; i++) {
239.                                                 Imgproc.line (rgbaMat, points [i], points [(i + 1) % 4], new Scalar (255, 0, 0, 255), 2);
240.                                         }
241.                
242.                                         Imgproc.rectangle (rgbaMat, roiRect.tl (), roiRect.br (), new Scalar (0, 255, 0, 255), 2);
243.  
244.  
245.  
246.  
247.                                         GameObject quad = gameObject;
248.                    
249.                                         //center point of detected face
250.                                         Point point2D = new Point (roiRect.x + roiRect.width / 2, roiRect.y + roiRect.height / 2);
251.                    
252.                                         float textureWidth = GetComponent<Renderer> ().material.mainTexture.width;
253.                                         float textureHeight = GetComponent<Renderer> ().material.mainTexture.height;
254.                    
255.                                         Matrix4x4 transCenterM =
256.                         Matrix4x4.TRS (new Vector3 (((float)point2D.x) - textureWidth / 2, (textureHeight - (float)point2D.y) - textureHeight / 2, 0), Quaternion.identity, new Vector3 (1, 1, 1));
257.                    
258.                    
259.                                         Vector3 translation = new Vector3 (quad.transform.localPosition.x, quad.transform.localPosition.y, quad.transform.localPosition.z);
260.                    
261.                                         Quaternion rotation =
262.                         Quaternion.Euler (quad.transform.localEulerAngles.x, quad.transform.localEulerAngles.y, quad.transform.localEulerAngles.z);
263.                    
264.                                         Vector3 scale = new Vector3 (quad.transform.localScale.x / textureWidth, quad.transform.localScale.y / textureHeight, 1);
265.                    
266.                                         Matrix4x4 trans2Dto3DM =
267.                         Matrix4x4.TRS (translation, rotation, scale);
268.                    
269.                                         Matrix4x4 resultM = trans2Dto3DM * transCenterM;
270.                    
271.                                         Vector3 point3DVec = new Vector3 (0, 0, 0);
272.                                         point3DVec = resultM.MultiplyPoint3x4 (point3DVec);
273.                    
274.                                         point3D.transform.position = point3DVec;
275.                                         point3D.transform.eulerAngles = gameObject.transform.eulerAngles;
276.  
277.  
278.                                 }
279.                
280.                                 Imgproc.putText (rgbaMat, "PLEASE TOUCH 4 POINTS", new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
281.  
282.                
283. //                Imgproc.putText (rgbaMat, "W:" + rgbaMat.width () + " H:" + rgbaMat.height () + " SO:" + Screen.orientation, new Point (5, rgbaMat.rows () - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
284.                
285.                                 Utils.matToTexture2D (rgbaMat, texture, colors);
286.  
287.  
288.  
289.  
290.                         }
291.  
292.                 }
293.    
294.                 /// <summary>
295.                 /// Raises the disable event.
296.                 /// </summary>
297.                 void OnDisable ()
298.                 {
299.                         webCamTextureToMatHelper.Dispose ();
300.                 }
301.  
302.                 /// <summary>
303.                 /// Raises the back button event.
304.                 /// </summary>
305.                 public void OnBackButton ()
306.                 {
307.                         #if UNITY_5_3 || UNITY_5_3_OR_NEWER
308.             SceneManager.LoadScene ("OpenCVForUnitySample");
309.                         #else
310.                         Application.LoadLevel ("OpenCVForUnitySample");
311.                         #endif
312.                 }
313.  
314.                 /// <summary>
315.                 /// Raises the play button event.
316.                 /// </summary>
317.                 public void OnPlayButton ()
318.                 {
319.                         webCamTextureToMatHelper.Play ();
320.                 }
321.  
322.                 /// <summary>
323.                 /// Raises the pause button event.
324.                 /// </summary>
325.                 public void OnPauseButton ()
326.                 {
327.                         webCamTextureToMatHelper.Pause ();
328.                 }
329.  
330.                 /// <summary>
331.                 /// Raises the stop button event.
332.                 /// </summary>
333.                 public void OnStopButton ()
334.                 {
335.                         webCamTextureToMatHelper.Stop ();
336.                 }
337.  
338.                 /// <summary>
339.                 /// Raises the change camera button event.
340.                 /// </summary>
341.                 public void OnChangeCameraButton ()
342.                 {
343.                         webCamTextureToMatHelper.Init (null, webCamTextureToMatHelper.requestWidth, webCamTextureToMatHelper.requestHeight, !webCamTextureToMatHelper.requestIsFrontFacing);
344.                 }
345.  
346.                 /// <summary>
347.                 /// Converts the screen point.
348.                 /// </summary>
349.                 /// <returns>The screen point.</returns>
350.                 /// <param name="screenPoint">Screen point.</param>
351.                 /// <param name="quad">Quad.</param>
352.                 /// <param name="cam">Cam.</param>
353.                 static Point convertScreenPoint (Point screenPoint, GameObject quad, Camera cam)
354.                 {
355.                         Vector2 tl;
356.                         Vector2 tr;
357.                         Vector2 br;
358.                         Vector2 bl;
359.                    
360.  
361.                         tl = cam.WorldToScreenPoint (new Vector3 (quad.transform.localPosition.x - quad.transform.localScale.x / 2, quad.transform.localPosition.y + quad.transform.localScale.y / 2, quad.transform.localPosition.z));
362.                         tr = cam.WorldToScreenPoint (new Vector3 (quad.transform.localPosition.x + quad.transform.localScale.x / 2, quad.transform.localPosition.y + quad.transform.localScale.y / 2, quad.transform.localPosition.z));
363.                         br = cam.WorldToScreenPoint (new Vector3 (quad.transform.localPosition.x + quad.transform.localScale.x / 2, quad.transform.localPosition.y - quad.transform.localScale.y / 2, quad.transform.localPosition.z));
364.                         bl = cam.WorldToScreenPoint (new Vector3 (quad.transform.localPosition.x - quad.transform.localScale.x / 2, quad.transform.localPosition.y - quad.transform.localScale.y / 2, quad.transform.localPosition.z));
365.  
366.  
367.                         Mat srcRectMat = new Mat (4, 1, CvType.CV_32FC2);
368.                         Mat dstRectMat = new Mat (4, 1, CvType.CV_32FC2);
369.  
370.                        
371.                         srcRectMat.put (0, 0, tl.x, tl.y, tr.x, tr.y, br.x, br.y, bl.x, bl.y);
372.                         dstRectMat.put (0, 0, 0.0, 0.0, quad.transform.localScale.x, 0.0, quad.transform.localScale.x, quad.transform.localScale.y, 0.0, quad.transform.localScale.y);
373.  
374.                        
375.                         Mat perspectiveTransform = Imgproc.getPerspectiveTransform (srcRectMat, dstRectMat);
376.  
377. //                        Debug.Log ("srcRectMat " + srcRectMat.dump ());
378. //                        Debug.Log ("dstRectMat " + dstRectMat.dump ());
379. //                        Debug.Log ("perspectiveTransform " + perspectiveTransform.dump ());
380.  
381.                         MatOfPoint2f srcPointMat = new MatOfPoint2f (screenPoint);
382.                         MatOfPoint2f dstPointMat = new MatOfPoint2f ();
383.  
384.                         Core.perspectiveTransform (srcPointMat, dstPointMat, perspectiveTransform);
385.  
386. //                        Debug.Log ("srcPointMat " + srcPointMat.dump ());
387. //                        Debug.Log ("dstPointMat " + dstPointMat.dump ());
388.  
389.                         return dstPointMat.toArray () [0];
390.                 }
391.         }
392. }

 

The Markerbased AR Sample is no longer working under unity 5.4. I just created a fresh project, and imported opencv and the marker based ar example. I get these errors now:

Could not start graph
UnityEngine.WebCamTexturelay()
MarkerBasedARSample.<init>c__Iterator0:MoveNext() (at Assets/MarkerBasedARSample/Scripts/WebCamTextureToMatHelper.cs:186)
UnityEngine.MonoBehaviour:StartCoroutine(IEnumerator)
MarkerBasedARSample.WebCamTextureToMatHelper:Init() (at Assets/MarkerBasedARSample/Scripts/WebCamTextureToMatHelper.cs:115)
MarkerBasedARSample.WebCamTextureMarkerBasedARSample:Start() (at Assets/MarkerBasedARSample/Scripts/WebCamTextureMarkerBasedARSample.cs:92)

Could not pause pControl
UnityEngine.WebCamTexturelay()
MarkerBasedARSample.<init>c__Iterator0:MoveNext() (at Assets/MarkerBasedARSample/Scripts/WebCamTextureToMatHelper.cs:186)
UnityEngine.MonoBehaviour:StartCoroutine(IEnumerator)
MarkerBasedARSample.WebCamTextureToMatHelper:Init() (at Assets/MarkerBasedARSample/Scripts/WebCamTextureToMatHelper.cs:115)
MarkerBasedARSample.WebCamTextureMarkerBasedARSample:Start() (at Assets/MarkerBasedARSample/Scripts/WebCamTextureMarkerBasedARSample.cs:92)

 

This error seems to be caused by WebCam device.
https://developer.vuforia.com/forum...scussion/critical-error-could-not-start-graph

 

Code (CSharp):

1.  
2.  
3. //_tex = inited Camera texture
4.  
5. ========================
6.  
7. _tmpMat = new Mat(
8.             _tex.height,
9.             _tex.width,
10.             CvType.CV_8UC4
11.         );
12.  
13. ========================
14.  
15. Utils.textureToMat(_camTex, _tmpMat);
16.  
17. Imgproc.putText (_tmpMat, "Hello!", new Point ((double)5, (double)_tmpMat.rows() - 10), Core.FONT_HERSHEY_SIMPLEX, 1.0, new Scalar (255, 255, 255, 255), 2, Imgproc.LINE_AA, false);
18.  
19. Texture2D texture = new Texture2D (_tmpMat.cols (), _tmpMat.rows (), TextureFormat.RGBA32, false);
20. Utils.matToTexture2D (_tmpMat, texture);
21.  
22. _outRen.material.mainTexture = (Texture)texture;

Where is my mistake?
I just want to get texture from camera, print "Hello!" and set as Texture of right Rect.

[Build and run from android]


UPD!
In release build mode all is good.

 

How can I change the SimpleBlob FeatureDetector params in realtime? Only with .YML?

http://docs.opencv.org/trunk/d8/da7/structcv_1_1SimpleBlobDetector_1_1Params.html#gsc.tab=0

 

Hey ya'll,

Question: I'm using the WebCamTextureAR sample, and it works great. The problem is that on my mobile device, the aspect ratio is 4:3 and I would really love if it would fill up the screen (16:9). I can get the quad to resize but I can't figure out how to get the AR objects to update accordingly. If anyone has any insight on how to either scale up the quad and match the AR objects to the larger image, or to scale the camera to reflect this change, that would be super super super helpful!

 

Upload a screenshot please. Visually it will be possible to better understand your question.

 

FeatureDetector params can be changed from .yml file only.

 

Thank you, that was perfect !
IPA created without error !

 

Hi - have you implement Tesseract for OpenCV for Unity yet http://docs.opencv.org/trunk/d7/ddc/classcv_1_1text_1_1OCRTesseract.html

 

Could you test the functionality of the boot options?
I'm working on a version of Unity 5.4.0.
Launch BlobDetektor and nothing happens.
Remove _detector.read (Utils.getFilePath ("blobparams.yml")); (it turns out that the detector takes the default settings in the code)
Only then the detector finds objects.

Your YAML file is different from the design with the
https://docs.unity3d.com/Manual/YAMLSceneExample.html

 

I am trying to implement Tesseract for OpenCV for Unity. However, I don't know whether that will be successful or not.

 

result screenshot on my enviroment.
Unity 5.0.4f3
OpenCVforUnity 2.0.5



if remove blobDetector.read (Utils.getFilePath ("blobparams.yml"));


Also, Format of yaml file Please refer to the information of OpenCV Java.

 

tesserect works on emgucv plugin for unity - your competitor. so, there should be a way.

 

Hi
can i implement SURF detector using this asset, i need to apply surf or sift in unity project .
Here is a link example using emgucv http://www.emgu.com/wiki/index.php/SURF_feature_detector_in_CSharp#Source_Code
i want this work in my unity project . im stuck here so please help me . Thank you

 

Hi,

I'm trying to build an iOS project in Unity Cloud, everything appears to be ok, but when I have to use your plugin the screen goes black.

I have no problem building in xCode, the plugin works fine.

Can you help me?

Thanks in advance.

 

Hi there, I have wrote the code for detection of rectangular objects corners from webcam, and obtained 4 points each frame, but the points are not really stable ( shakes), so I need some method to reduce those shakes and I found out that Kalman filter is great for this. Unfortunately I couldn't translate cpp examples properly. I use the opencv for unity wrapper. Please, help to deal with it. I need some example of implementing of Kalman filter based on this wrapper or c#(without opencv) or java code. Any help will appreciated.

 

When I build OpenCVforUnitySample in Unity Cloud Build before v2.0.6 release, such problems did not occur.
Could you tell me the ios setting of UnityCloudBuild?
Unity version
Xcode version

 

This is what the default FaceLandmarkDetector example looks like– basically, I'm trying to get rid of the blue space. The iPhone shoots at 4:3 aspect ratio, so there's space above and below. I can hardcode scale the Main Camera and AR Camera but I'd like to do it dynamically, based on the screen dimensions of the phone.

 

Do you want to stream to accommodate screen sizes?
If so, then I do through the code. (Stretching height relative to the width) to the height of the screen.

 

Hi Enox & all,

I'm attempting to use grabcut using a Mask, as shown here : http://docs.opencv.org/3.1.0/d8/d83/tutorial_py_grabcut.html

I am using this as a model :
http://stackoverflow.com/questions/14111716/how-to-set-a-mask-image-for-grabcut-in-opencv

I had success with grabcut by initializing with a rect, but the INIT_WITH_MASK version does not seem to be working.

The top image shows INIT_WITH_RECT working.



Below, is an attempt at Grabcut INIT_WITH_MASK. It is possible that I am not understanding how this is used, but this is the result after passing a the resulting image from version one as well as a mask that I drew manually, see the bottom square in this image. To me, this looks like it is just masking out the pixels, not reapplying the algorithm :





Without digging too deep, here is the code for the second iteration of the algorithm. Any advice?

Code (CSharp):

1. public void Grab()
2.     {
3.         Texture2D sourceImage = Resources.Load (sourceImg0) as Texture2D;
4.         one.material.mainTexture = sourceImage;
5.         Mat image = new Mat(sourceImage.height, sourceImage.width, CvType.CV_8UC3);
6.         Utils.texture2DToMat(sourceImage, image);
7.      
8.         //http://stackoverflow.com/questions/30872353/optimizing-performance-of-grabcut-in-opencv-java
9.         //OpenCVForUnity.Rect rect = new OpenCVForUnity.Rect(image.width () / 4, image.height () / 4, 3 * image.width () / 4, 3 * image.height () / 4);
10.         OpenCVForUnity.Rect rect = new OpenCVForUnity.Rect(0, 0, 9 * image.width () / 10, 9 * image.height () / 10);
11.         Mat gCutResult = new Mat ();
12.         Mat bgdModel = new Mat ();
13.         Mat fgModel = new Mat ();
14.         Mat source = new Mat (1, 1, CvType.CV_8U, new Scalar (3));
15.         Imgproc.grabCut (image, gCutResult, rect, bgdModel, fgModel, 2, Imgproc.GC_INIT_WITH_RECT);
16.  
17.         maskRend0.material.mainTexture = ReturnTextureFromMat (gCutResult);
18.  
19.         Core.compare (gCutResult, source, gCutResult, Core.CMP_EQ);
20.         Mat foreground = new Mat (image.size (), CvType.CV_8UC3, new Scalar (255, 255, 255));
21.         image.copyTo (foreground, gCutResult);
22.      
23.         //do we need this step?
24.         Imgproc.cvtColor(foreground, foreground, Imgproc.COLOR_BGR2BGRA);
25.      
26.         mergeResult.material.mainTexture = ConvertMatToAlphaTexture (foreground);
27.         //foreground = ConvertMatToAlphaMat (foreground);
28.  
29.         firstResult = foreground;
30.  
31.         //try some different texture formats..
32.         Texture2D resultTexture = new Texture2D(firstResult.cols(), firstResult.rows(), TextureFormat.RGBA32, false);
33.         Utils.matToTexture2D (firstResult, resultTexture);
34.         maskRend.material.mainTexture = resultTexture;
35.      
36.     }
37.  
38.     /// <summary>
39.     /// This version is to be used after initial grabcut
40.     /// </summary>
41.     /// <param name="image">Image.</param>
42.     public void GrabMaskFromButton2()
43.     {
44.         Mat image = new Mat ();
45.  
46.         if (string.IsNullOrEmpty (sourceImg1)) {
47.                      image = firstResult;
48.                     one.material.mainTexture = ReturnTextureFromMat (image);
49.         } else {
50.  
51.             Texture2D sourceImage = Resources.Load (sourceImg1) as Texture2D;
52.             one.material.mainTexture = sourceImage;
53.              image = new Mat (sourceImage.height, sourceImage.width, CvType.CV_8UC3);
54.             Utils.texture2DToMat (sourceImage, image);
55.         }
56.  
57.  
58.  
59.  
60.         Texture2D maskImage = Resources.Load (maskImg0) as Texture2D;
61.         Mat maskMat = new Mat(maskImage.width, maskImage.height, CvType.CV_8U,new Scalar( 0)) ; //CV_8UC3  //CV_16S //CV_8UC1
62.         Utils.texture2DToMat(maskImage, maskMat);
63.         maskRend0.material.mainTexture = ReturnTextureFromMat (maskMat);
64.         ConvertToOpencvValues (maskMat);
65.         //PrintMatProperties (maskMat);
66.  
67.         //http://stackoverflow.com/questions/30872353/optimizing-performance-of-grabcut-in-opencv-java
68.         //OpenCVForUnity.Rect rect = new OpenCVForUnity.Rect(image.width () / 4, image.height () / 4, 3 * image.width () / 4, 3 * image.height () / 4);
69.         OpenCVForUnity.Rect rect = new OpenCVForUnity.Rect(0, 0, 9 * image.width () / 10, 9 * image.height () / 10);
70.         //Mat gCutResult = new Mat ();
71.         Mat bgdModel = new Mat ();
72.         Mat fgModel = new Mat ();
73.  
74.         Imgproc.grabCut (image, maskMat, rect, bgdModel, fgModel, 1, Imgproc.GC_INIT_WITH_MASK); //GC_INIT_WITH_MASK
75.  
76.         maskRend.material.mainTexture = ReturnTextureFromMat (maskMat);
77.  
78.         mergeResult.material.mainTexture = null;
79.  
80.         convertToHumanValues (maskMat);
81.         //Imgproc.threshold (maskMat, maskMat, 128, 255, Imgproc.THRESH_TOZERO);
82.  
83.         Mat foreground = new Mat(image.size(), CvType.CV_8UC1, new Scalar(0, 0, 0));
84.         image.copyTo (foreground, maskMat);
85.  
86.         two.material.mainTexture = ReturnTextureFromMat (foreground);
87.  
88.     }
89.  
90.     private static void convertToHumanValues(Mat mask) {
91.         byte[] buffer = new byte[3];
92.         for (int x = 0; x < mask.rows(); x++) {
93.             for (int y = 0; y < mask.cols(); y++) {
94.                 mask.get(x, y, buffer);
95.                 int value = buffer[0];
96.                 if (value == Imgproc.GC_BGD) {
97.                     buffer[0] = 0; // for sure background
98.                 } else if (value == Imgproc.GC_PR_BGD) {
99.                     buffer[0] = 85; // probably background
100.                 } else if (value == Imgproc.GC_PR_FGD) {
101.                     buffer[0] = (byte) 170; // probably foreground
102.                 } else {
103.                     buffer[0] = (byte) 255; // for sure foreground
104.  
105.                 }
106.                 mask.put(x, y, buffer);
107.             }
108.         }
109.     }
110.  
111.  
112.     private static void ConvertToOpencvValues(Mat mask) {
113.         byte[] buffer = new byte[3];
114.         for (int x = 0; x < mask.rows(); x++) {
115.             for (int y = 0; y < mask.cols(); y++) {
116.                 mask.get(x, y, buffer);
117.                 int value = buffer[0];
118.                 //if 0, do nothing...
119. //                if (value == 0) {
120. //                    print ("Doing nothing");
121.                 //                }else
122.                 if (value > 0 && value < 64) {
123.                     buffer[0] = Imgproc.GC_BGD; // for sure background
124.                     //print("BGD");
125.                 } else if (value >= 64 && value < 128) {
126.                     buffer[0] = Imgproc.GC_PR_BGD; // probably background
127.                     //print("PBGD");
128.                 } else if (value >= 128 && value < 192) {
129.                     buffer[0] = Imgproc.GC_PR_FGD; // probably foreground
130.                     //print("PFGD");
131.                 } else {
132.                     buffer[0] = Imgproc.GC_FGD; // for sure foreground
133.                     //print("FGD");
134.                 }
135.                 mask.put(x, y, buffer);
136.             }
137.         }
138.     }

 

I got it working pretty easily. There's a lot of code there to go through to see any problems. It's weird the mask that gets returned is 0 == background, 1 == foreground, 2 == background 3 == foreground. So I can't think of a nice way to create an alpha mask without doing in manually. Advice would be great. Is there a even value compare?

Code (csharp):

1.    
2. public Mat GrabCut(Mat image, Mat mask) {
3.         Rect rectangle = new Rect(0, 0, image.cols(), image.rows());
4.  
5.         Imgproc.grabCut(image, mask, rectangle, new Mat(), new Mat(), 8, Imgproc.GC_INIT_WITH_MASK);
6.         Mat result = new Mat(mask.rows(), mask.cols(), CvType.CV_8UC4);
7.         copyMaskIntoResult(mask, image, result);
8.         return result;
9. }
10.  
11. // copies img into result where the mask is foreground otherwise copies 0,0,0,0 pixel
12. static void copyMaskIntoResult(Mat mask, Mat img, Mat result) {
13.         UnityEngine.Assertions.Assert.IsTrue(mask.rows() == img.rows() && mask.cols() == img.cols());
14.         byte[] buffer = new byte[3];
15.         byte[] impix = new byte[4];
16.         byte[] bgpix = new byte[4] { 0, 0, 0, 0 };
17.  
18.         for (int r = 0; r < mask.rows(); r++) {
19.             for (int c = 0; c < mask.cols(); c++) {
20.                 mask.get(r, c, buffer);
21.                 int value = buffer[0];
22.                 if (value == Imgproc.GC_BGD || value == Imgproc.GC_PR_BGD) {
23.                     result.put(r, c, bgpix);
24.                 }
25.                 else {
26.                     img.get(r, c, buffer);
27.                     impix[0] = buffer[0];
28.                     impix[1] = buffer[1];
29.                     impix[2] = buffer[2];
30.                     impix[3] = 255;
31.                     result.put(r, c, impix);
32.                 }
33.             }
34.         }
35.     }
36.  

 

nykwil - did you get a result that looks like the grabcut algorithm is working? Can you post your result images?

 

here it is

 

Please try the following code.(changed to scale Quad to the entire screen.)

Code (CSharp):

1. /// <summary>
2.         /// Raises the web cam texture to mat helper inited event.
3.         /// </summary>
4.         public void OnWebCamTextureToMatHelperInited ()
5.         {
6.             Debug.Log ("OnWebCamTextureToMatHelperInited");
7.            
8.             Mat webCamTextureMat = webCamTextureToMatHelper.GetMat ();
9.            
10.             colors = new Color32[webCamTextureMat.cols () * webCamTextureMat.rows ()];
11.             texture = new Texture2D (webCamTextureMat.cols (), webCamTextureMat.rows (), TextureFormat.RGBA32, false);
12.  
13.             gameObject.transform.localScale = new Vector3 (webCamTextureMat.cols (), webCamTextureMat.rows (), 1);
14.             Debug.Log ("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
15.  
16.             float width = gameObject.transform.localScale.x;
17.             float height = gameObject.transform.localScale.y;
18.  
19.             float imageScale = 1.0f;
20.             float widthScale = (float)Screen.width / width;
21.             float heightScale = (float)Screen.height / height;
22.             /*
23.             if (widthScale < heightScale)
24.             {
25.                 Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
26.                 imageScale = (float)Screen.height / (float)Screen.width;
27.             }
28.             else
29.             {
30.                 Camera.main.orthographicSize = height / 2;
31.             }
32.             */
33.  
34.             if (widthScale < heightScale) {
35.                 Camera.main.orthographicSize = height / 2;
36.                 imageScale = 0.5f;
37.             } else {
38.                 Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
39.                 imageScale = (float)Screen.height / (float)Screen.width / 0.5f;
40.             }
41.            
42.  
43.  
44.             gameObject.GetComponent<Renderer> ().material.mainTexture = texture;
45.                                    
46.                                    
47.             int max_d = Mathf.Max (webCamTextureMat.rows (), webCamTextureMat.cols ());
48.             camMatrix = new Mat (3, 3, CvType.CV_64FC1);
49.             camMatrix.put (0, 0, max_d);
50.             camMatrix.put (0, 1, 0);
51.             camMatrix.put (0, 2, webCamTextureMat.cols () / 2.0f);
52.             camMatrix.put (1, 0, 0);
53.             camMatrix.put (1, 1, max_d);
54.             camMatrix.put (1, 2, webCamTextureMat.rows () / 2.0f);
55.             camMatrix.put (2, 0, 0);
56.             camMatrix.put (2, 1, 0);
57.             camMatrix.put (2, 2, 1.0f);
58.                                    
59.             Size imageSize = new Size (webCamTextureMat.cols () * imageScale, webCamTextureMat.rows () * imageScale);
60.             double apertureWidth = 0;
61.             double apertureHeight = 0;
62.             double[] fovx = new double[1];
63.             double[] fovy = new double[1];
64.             double[] focalLength = new double[1];
65.             Point principalPoint = new Point ();
66.             double[] aspectratio = new double[1];
67.                                    
68.                                    
69.                                    
70.                                    
71.             Calib3d.calibrationMatrixValues (camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio);
72.                                    
73.             Debug.Log ("imageSize " + imageSize.ToString ());
74.             Debug.Log ("apertureWidth " + apertureWidth);
75.             Debug.Log ("apertureHeight " + apertureHeight);
76.             Debug.Log ("fovx " + fovx [0]);
77.             Debug.Log ("fovy " + fovy [0]);
78.             Debug.Log ("focalLength " + focalLength [0]);
79.             Debug.Log ("principalPoint " + principalPoint.ToString ());
80.             Debug.Log ("aspectratio " + aspectratio [0]);
81.                                    
82.                                    
83.             if (widthScale < heightScale) {
84.                 ARCamera.fieldOfView = (float)fovx [0];
85.             } else {
86.                 ARCamera.fieldOfView = (float)fovy [0];
87.             }
88.  
89.                                    
90.             Debug.Log ("camMatrix " + camMatrix.dump ());
91.                                    
92.                                    
93.             distCoeffs = new MatOfDouble (0, 0, 0, 0);
94.             Debug.Log ("distCoeffs " + distCoeffs.dump ());
95.                                    
96.                                    
97.                                    
98.             invertYM = Matrix4x4.TRS (Vector3.zero, Quaternion.identity, new Vector3 (1, -1, 1));
99.             Debug.Log ("invertYM " + invertYM.ToString ());
100.            
101.             invertZM = Matrix4x4.TRS (Vector3.zero, Quaternion.identity, new Vector3 (1, 1, -1));
102.             Debug.Log ("invertZM " + invertZM.ToString ());
103.            
104.            
105.             axes.SetActive (false);
106.             head.SetActive (false);
107.             rightEye.SetActive (false);
108.             leftEye.SetActive (false);
109.             mouth.SetActive (false);
110.  
111.  
112.             mouthParticleSystem = mouth.GetComponentsInChildren<ParticleSystem> (true);
113.  
114.         }

 

Hi Nykwil,

I used your code but the result is the same as my previous attempt. Using your code directly, I received this error :
CvException: Provided data element number (3) should be multiple of the Mat channels count (4)
OpenCVForUnity.Mat.get (Int32 row, Int32 col, System.Byte[] data) (at Assets/OpenCVForUnity/org/opencv/core/Mat.cs:3071)
InkmageGrab.copyMaskIntoResult (OpenCVForUnity.Mat mask, OpenCVForUnity.Mat img, OpenCVForUnity.Mat result) (at Assets/InkmageGrab.cs:163)

Which I fixed by changing : byte[]buffer=newbyte[3]; to byte[]buffer=newbyte[4];

Does that mean my mask is not initialized correctly? How was your mask initialized and what is the result?

Thanks for looking into it,
Bryan

 

By the way,
The way I read into this algorithm is that the second pass initializing with a mask, is not supposed to just 'mask out' the image. It is supposed to help in removing or adding pieces that were before missed by the algorithm. Right now, this code is merely masking things out.

Please let me know if I am misunderstanding?
Bryan

 

Hello, I bought your asset today and I was wondering if I can have an x amount of AR Objects display at the same time depending on the number of faces tracked using the FaceTracker AR Sample, and if so, could you tell me what to change in the original code? Thanks in advance

 

Hello, I just bought your asset and I can not find a method findHomography. Can you explain or give any other alternative? I do not even find a homography example or tutorial on your assets. Thanks in advance

 

These settings might be the cause of the problem.
Build Asset Bundles = No
Copy to Streaming Assets = No
The other my settings are the same as your settings.

 

Code (CSharp):

1. using UnityEngine;
2. using System.Collections;
3.  
4. #if UNITY_5_3 || UNITY_5_3_OR_NEWER
5. using UnityEngine.SceneManagement;
6. #endif
7. using OpenCVForUnity;
8.  
9. namespace OpenCVForUnitySample
10. {
11.     /// <summary>
12.     /// Texture2D to mat sample.
13.     /// </summary>
14.     public class GrabCutSample : MonoBehaviour
15.     {
16.  
17.         // Use this for initialization
18.         void Start ()
19.         {
20.  
21.             Texture2D imageTexture = Resources.Load ("lena_grabcut") as Texture2D;
22.  
23.             Mat image = new Mat (imageTexture.height, imageTexture.width, CvType.CV_8UC3);
24.  
25.             Utils.texture2DToMat (imageTexture, image);
26.             Debug.Log ("image.ToString " + image.ToString ());
27.  
28.             Texture2D maskTexture = Resources.Load ("lena_grabcut_mask") as Texture2D;
29.  
30.             Mat mask = new Mat (imageTexture.height, imageTexture.width, CvType.CV_8UC1);
31.  
32.             Utils.texture2DToMat (maskTexture, mask);
33.             Debug.Log ("mask.ToString " + mask.ToString ());
34.  
35.  
36.  
37.             OpenCVForUnity.Rect rectangle = new OpenCVForUnity.Rect (10, 10, image.cols () - 20, image.rows () - 20);
38.  
39.             Mat bgdModel = new Mat (); // extracted features for background
40.             Mat fgdModel = new Mat (); // extracted features for foreground
41.             Mat source = new Mat (1, 1, CvType.CV_8U, new Scalar (0));
42.  
43.             convertToOpencvValues (mask); // from human readable values to OpenCV values
44.  
45.             int iterCount = 1;
46. //            Imgproc.grabCut (image, mask, rectangle, bgdModel, fgdModel, iterCount, Imgproc.GC_INIT_WITH_RECT);
47.             Imgproc.grabCut (image, mask, rectangle, bgdModel, fgdModel, iterCount, Imgproc.GC_INIT_WITH_MASK);
48.  
49.  
50.             convertToHumanValues (mask); // back to human readable values
51.             Imgproc.threshold (mask, mask, 128, 255, Imgproc.THRESH_TOZERO);
52.  
53.             Mat foreground = new Mat (image.size (), CvType.CV_8UC1, new Scalar (0, 0, 0));
54.             image.copyTo (foreground, mask);
55.  
56.  
57.             Texture2D texture = new Texture2D (image.cols (), image.rows (), TextureFormat.RGBA32, false);
58.  
59.             Utils.matToTexture2D (foreground, texture);
60.  
61.             gameObject.GetComponent<Renderer> ().material.mainTexture = texture;
62.  
63.         }
64.  
65.         private static void convertToHumanValues (Mat mask)
66.         {
67.             byte[] buffer = new byte[3];
68.             for (int x = 0; x < mask.rows (); x++) {
69.                 for (int y = 0; y < mask.cols (); y++) {
70.                     mask.get (x, y, buffer);
71.                     int value = buffer [0];
72.                     if (value == Imgproc.GC_BGD) {
73.                         buffer [0] = 0; // for sure background
74.                     } else if (value == Imgproc.GC_PR_BGD) {
75.                         buffer [0] = 85; // probably background
76.                     } else if (value == Imgproc.GC_PR_FGD) {
77.                         buffer [0] = (byte)170; // probably foreground
78.                     } else {
79.                         buffer [0] = (byte)255; // for sure foreground
80.  
81.                     }
82.                     mask.put (x, y, buffer);
83.                 }
84.             }
85.         }
86.  
87.         /**
88.      * Converts level of grayscale into OpenCV values. White - foreground, Black
89.      * - background.
90.      *
91.      * @param mask
92.      */
93.         private static void convertToOpencvValues (Mat mask)
94.         {
95.             byte[] buffer = new byte[3];
96.             for (int x = 0; x < mask.rows (); x++) {
97.                 for (int y = 0; y < mask.cols (); y++) {
98.                     mask.get (x, y, buffer);
99.                     int value = buffer [0];
100.                     if (value >= 0 && value < 64) {
101.                         buffer [0] = Imgproc.GC_BGD; // for sure background
102.                     } else if (value >= 64 && value < 128) {
103.                         buffer [0] = Imgproc.GC_PR_BGD; // probably background
104.                     } else if (value >= 128 && value < 192) {
105.                         buffer [0] = Imgproc.GC_PR_FGD; // probably foreground
106.                     } else {
107.                         buffer [0] = Imgproc.GC_FGD; // for sure foreground
108.  
109.                     }
110.                     mask.put (x, y, buffer);
111.                 }
112.             }
113.  
114.         }
115.  
116.    
117.         // Update is called once per frame
118.         void Update ()
119.         {
120.    
121.         }
122.  
123.         public void OnBackButton ()
124.         {
125.             #if UNITY_5_3 || UNITY_5_3_OR_NEWER
126.             SceneManager.LoadScene ("OpenCVForUnitySample");
127.             #else
128.                         Application.LoadLevel ("OpenCVForUnitySample");
129.             #endif
130.         }
131.  
132.     }
133. }

Imgproc.GC_INIT_WITH_RECT

Imgproc.GC_INIT_WITH_MASK

 

Checking on the status of OCR Tesseract for OpenCV for Unity package

 

findHomography() has been implemented in OpenCV for Unity.
http://enoxsoftware.github.io/OpenC...alib3d.html#a4fffd0de09e28dacbf1c7e65d8b3544b
Please refer to the usage example of OpenCV Java.

 

Hi Enox,

If you notice - this example actually just masks the image. It doesn't do a grabcut. Not the expected result.

If you take a look at the OpenCV docs here, near the bottom of the page... : http://docs.opencv.org/3.1.0/d8/d83/tutorial_py_grabcut.html

You can see that the user drew a white line across a small area of the hair (not the whole hair, precisely - for that would be a mask). A grey line around the grass. He did not draw a line across all of the grass, just a line. Drawing around the whole grass would be a mask. This algorithm isn't supposed to simply mask things.

 

It seems to need to set "Imgproc.GC_PR_BGD" to the background Area.

Code (CSharp):

1. using UnityEngine;
2. using System.Collections;
3.  
4. #if UNITY_5_3 || UNITY_5_3_OR_NEWER
5. using UnityEngine.SceneManagement;
6. #endif
7. using OpenCVForUnity;
8.  
9. namespace OpenCVForUnitySample
10. {
11.     /// <summary>
12.     /// Texture2D to mat sample.
13.     /// </summary>
14.     public class GrabCutSample : MonoBehaviour
15.     {
16.  
17.         // Use this for initialization
18.         void Start ()
19.         {
20.  
21. //            Texture2D imageTexture = Resources.Load ("lena_grabcut") as Texture2D;
22.             Texture2D imageTexture = Resources.Load ("messi5") as Texture2D;
23.  
24.             Mat image = new Mat (imageTexture.height, imageTexture.width, CvType.CV_8UC3);
25.  
26.             Utils.texture2DToMat (imageTexture, image);
27.             Debug.Log ("image.ToString " + image.ToString ());
28.  
29. //            Texture2D maskTexture = Resources.Load ("lena_grabcut_mask") as Texture2D;
30.             Texture2D maskTexture = Resources.Load ("messi5_mask") as Texture2D;
31.  
32.             Mat mask = new Mat (imageTexture.height, imageTexture.width, CvType.CV_8UC1);
33.  
34.             Utils.texture2DToMat (maskTexture, mask);
35.             Debug.Log ("mask.ToString " + mask.ToString ());
36.  
37.  
38.  
39.             OpenCVForUnity.Rect rectangle = new OpenCVForUnity.Rect (10, 10, image.cols () - 20, image.rows () - 20);
40.  
41.             Mat bgdModel = new Mat (); // extracted features for background
42.             Mat fgdModel = new Mat (); // extracted features for foreground
43.             Mat source = new Mat (1, 1, CvType.CV_8U, new Scalar (0));
44.  
45.             convertToOpencvValues (mask); // from human readable values to OpenCV values
46.  
47.             int iterCount = 5;
48. //            Imgproc.grabCut (image, mask, rectangle, bgdModel, fgdModel, iterCount, Imgproc.GC_INIT_WITH_RECT);
49.             Imgproc.grabCut (image, mask, rectangle, bgdModel, fgdModel, iterCount, Imgproc.GC_INIT_WITH_MASK);
50.  
51.  
52.  
53.             convertToHumanValues (mask); // back to human readable values
54.             Imgproc.threshold (mask, mask, 128, 255, Imgproc.THRESH_TOZERO);
55.  
56.             Mat foreground = new Mat (image.size (), CvType.CV_8UC3, new Scalar (0, 0, 0));
57.             image.copyTo (foreground, mask);
58.  
59.  
60.             Texture2D texture = new Texture2D (image.cols (), image.rows (), TextureFormat.RGBA32, false);
61.  
62.             Utils.matToTexture2D (foreground, texture);
63.  
64.             gameObject.GetComponent<Renderer> ().material.mainTexture = texture;
65.  
66.         }
67.  
68.         private static void convertToHumanValues (Mat mask)
69.         {
70.             byte[] buffer = new byte[1];
71.             for (int x = 0; x < mask.rows (); x++) {
72.                 for (int y = 0; y < mask.cols (); y++) {
73.                     mask.get (x, y, buffer);
74.                     int value = buffer [0];
75.                     if (value == Imgproc.GC_BGD) {
76.                         buffer [0] = 0; // for sure background
77.                     } else if (value == Imgproc.GC_PR_BGD) {
78.                         buffer [0] = 85; // probably background
79.                     } else if (value == Imgproc.GC_PR_FGD) {
80.                         buffer [0] = (byte)170; // probably foreground
81.                     } else {
82.                         buffer [0] = (byte)255; // for sure foreground
83.  
84.                     }
85.                     mask.put (x, y, buffer);
86.                 }
87.             }
88.         }
89.  
90.         /**
91.      * Converts level of grayscale into OpenCV values. White - foreground, Black
92.      * - background.
93.      *
94.      * @param mask
95.      */
96.         private static void convertToOpencvValues (Mat mask)
97.         {
98.             byte[] buffer = new byte[1];
99.             for (int x = 0; x < mask.rows (); x++) {
100.                 for (int y = 0; y < mask.cols (); y++) {
101.                     mask.get (x, y, buffer);
102.                     int value = buffer [0];
103.                     if (value >= 0 && value < 64) {
104.                         buffer [0] = Imgproc.GC_BGD; // for sure background
105.                     } else if (value >= 64 && value < 128) {
106.                         buffer [0] = Imgproc.GC_PR_BGD; // probably background
107.                     } else if (value >= 128 && value < 192) {
108.                         buffer [0] = Imgproc.GC_PR_FGD; // probably foreground
109.                     } else {
110.                         buffer [0] = Imgproc.GC_FGD; // for sure foreground
111.  
112.                     }
113.                     mask.put (x, y, buffer);
114.                 }
115.             }
116.  
117.         }
118.  
119.    
120.         // Update is called once per frame
121.         void Update ()
122.         {
123.    
124.         }
125.  
126.         public void OnBackButton ()
127.         {
128.             #if UNITY_5_3 || UNITY_5_3_OR_NEWER
129.             SceneManager.LoadScene ("OpenCVForUnitySample");
130.             #else
131.                         Application.LoadLevel ("OpenCVForUnitySample");
132.             #endif
133.         }
134.  
135.     }
136. }
137.