Integrating Android Voice Recognition with my Cardboard Project

Hi,

I am working with the Google Carboard SDK, and I actually try to integrate the Android native voice recognition service.


Google Carboard SDK version 0.6
Unity version 5.3.1 f1


First question:

Is there any simple way to call the voice recognition service with the Unity Google Cardboard SDK, without making any Android custom project? I found no information about it in the dev guide, but the SDK being made by Google, who has awesome voice recognition API and the voice interface being a potentially fantastic interface for Virtual Reality… I don’t know, may be I missed something?


Second question:

To integrate the Voice Speech Recognition in my android project I’ve created a RecognitionService java class, with a static function (
public static void StartListenning(GoogleUnityActivity activity)
) in, starting the service. At the first interaction, my C# call the StartListenning function, launch the voice service works perfectly, and send back to Unity C# your text. But the second time i try to relaunch the service, nothing. StartListenning isn’t called and only look like to be called only when the UnityPlayer has been stopped.

The StartListenning function IS called, but the RecognitionService doesn't look to be launched... I think I will checkout how the lifecycle of Android service are working next week.

Any ideas why my StartListenning isn't working for his second call?

My C# code:

Code (CSharp):

1.  AndroidJavaClass androidJC = new AndroidJavaClass("com.unity3d.player.UnityPlayer");
2.              AndroidJavaObject jo = androidJC.GetStatic<AndroidJavaObject>("currentActivity");
3.             AndroidJavaClass jc = new AndroidJavaClass("com.snuffchan.fps.SpeechService");
4.             jc.CallStatic("StartListenning", jo);
5.  

My Android manifest code:

Code (CSharp):

1.  <service android:name="com.snuffchan.fps.SpeechService" android:enabled="true">
2.       </service>

My JAVA service code:

Code (Boo):

1. package com.snuffchan.fps;
2.  
3. import android.app.Service;
4. import android.content.Intent;
5. import android.os.Bundle;
6. import android.speech.RecognitionListener;
7. import android.speech.RecognitionService;
8. import android.speech.RecognizerIntent;
9. import android.speech.SpeechRecognizer;
10. import android.util.Log;
11.  
12. import com.google.unity.GoogleUnityActivity;
13. import com.unity3d.player.UnityPlayer;
14.  
15. import java.util.ArrayList;
16.  
17. import static android.speech.SpeechRecognizer.ERROR_AUDIO;
18. import static android.speech.SpeechRecognizer.ERROR_CLIENT;
19. import static android.speech.SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS;
20. import static android.speech.SpeechRecognizer.ERROR_NETWORK;
21. import static android.speech.SpeechRecognizer.ERROR_NETWORK_TIMEOUT;
22. import static android.speech.SpeechRecognizer.ERROR_NO_MATCH;
23. import static android.speech.SpeechRecognizer.ERROR_RECOGNIZER_BUSY;
24. import static android.speech.SpeechRecognizer.ERROR_SERVER;
25. import static android.speech.SpeechRecognizer.ERROR_SPEECH_TIMEOUT;
26. import static android.speech.SpeechRecognizer.createSpeechRecognizer;
27.  
28. /**
29. * Created by alexandre on 16/02/16.
30. */
31. public class SpeechService extends RecognitionService implements RecognitionListener {
32.  
33.     private SpeechRecognizer m_EngineSR;
34.     //private Context mContext;
35.     public static android.app.Activity mActivity;
36.     static String TAG = "VOICE RECOGNITION";
37.  
38.     /**
39.      * StartListenning**************************************************
40.      * Static function call by the c# to launch the service SpeechService
41.      */
42.     public static void StartListenning(GoogleUnityActivity activity) {
43.         Log.i(TAG, "START THE SERVICE! ");
44.  
45.         if( UnityPlayer.currentActivity != null) {
46.             UnityPlayer.UnitySendMessage("CubeMessage", "GetMessage", "TRY 2 RESTART SERVICE");
47.             mActivity = activity;
48.             Intent intent = new Intent(mActivity, SpeechService.class);
49.             mActivity.startService(intent);
50.         }
51.  
52.     }
53.  
54.      //The service is created and voice recognition service has been started
55.     @Override
56.     public void onCreate() {
57.  
58.         Log.i(TAG, "onCreate()");
59.         m_EngineSR = createSpeechRecognizer(this);
60.         m_EngineSR.setRecognitionListener(this);
61.         Intent voiceIntent = RecognizerIntent.getVoiceDetailsIntent(getApplicationContext());
62.         m_EngineSR.startListening(voiceIntent);
63.         super.onCreate();
64.     }
65.  
66.     private void checkForCommands(Bundle bundle) {
67.         ArrayList<String> voiceText = bundle.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION);
68.         if (voiceText != null) {
69.             //SendToUnity("");
70.             if (voiceText.size() > 0) {
71.                 SendToUnity(voiceText.get(0));
72.             } else {
73.                 SendToUnity("nothing!");
74.             }
75.  
76.         } else {
77.             SendToUnity("voiceText empty!");
78.         }
79.     }
80.  
81.     @Override
82.     public int onStartCommand(Intent intent, int flags, int startId){
83.        return Service.START_NOT_STICKY;
84.     }
85.  
86.     /**
87.      * Send the data to Unity
88.      * @param text text or errors string
89.      */
90.     public void SendToUnity(String text){
91.         if(m_EngineSR!=null) {
92.  
93.             if(text != null && text.isEmpty() )Log.i("TESTING: ", "final message! =" + text);
94.             try {
95.  
96.                 if( UnityPlayer.currentActivity != null) UnityPlayer.UnitySendMessage("CubeMessage", "GetMessage", text);
97.  
98.             } catch (Exception e) {
99.                // Log.e(TAG, "UnitySendMessage failed" + e.getMessage());
100.             }
101.            // m_EngineSR.stopListening();
102.         }
103.  
104.  
105.         this.onDestroy();
106.     }
107.  
108.     /***
109.      * SERVICE STUFF
110.      */
111.  
112.  
113.     @Override
114.     public void onDestroy() {
115.         if(m_EngineSR!= null)m_EngineSR.cancel();
116.         Log.i("SimpleVoiceService", "Service stopped");
117.         super.onDestroy();
118.     }
119.  
120.     @Override
121.     protected void onStartListening(Intent recognizerIntent, Callback listener) {
122.  
123.     }
124.  
125.     @Override
126.     protected void onCancel(Callback listener) {
127.  
128.     }
129.  
130.     @Override
131.     protected void onStopListening(Callback listener) {
132.  
133.     }
134.  
135.  
136.     @Override
137.     public void onReadyForSpeech(Bundle params) {
138.  
139.     }
140.  
141.     @Override
142.     public void onBeginningOfSpeech() {
143.  
144.     }
145.  
146.     @Override
147.     public void onRmsChanged(float rmsdB){
148.     }
149.  
150.     @Override
151.     public void onBufferReceived(byte[] buffer) {
152.  
153.     }
154.  
155.     @Override
156.     public void onEndOfSpeech() {
157.  
158.     }
159.  
160.     @Override
161.     public void onError(int error) {
162.         try {
163.  
164.  
165.             String message;
166.             switch (error)
167.             {
168.                 case ERROR_AUDIO:
169.                     message = "Audio recording error";
170.                     break;
171.                 case ERROR_CLIENT:
172.                     message = "Client side error";
173.                     break;
174.                 case ERROR_INSUFFICIENT_PERMISSIONS:
175.                     message = "Insufficient permissions";
176.                     break;
177.                 case ERROR_NETWORK:
178.                     message = "Network error";
179.                     break;
180.                 case ERROR_NETWORK_TIMEOUT:
181.                     message = "Network timeout";
182.                     break;
183.                 case ERROR_NO_MATCH:
184.                     message = "No match";
185.                     break;
186.                 case ERROR_RECOGNIZER_BUSY:
187.                     message = "RecognitionService busy";
188.                     break;
189.                 case ERROR_SERVER:
190.                     message = "error from server";
191.                     break;
192.                 case ERROR_SPEECH_TIMEOUT:
193.                     message = "No speech input";
194.                     break;
195.                 default:
196.                     message = "Didn't understand, please try again.";
197.                     break;
198.             }
199.  
200.             SendToUnity(message);
201.  
202.         } catch (Exception e) {
203.             e.printStackTrace();
204.         }
205.         //m_SRListener.onError(error);
206.     }
207.  
208.     @Override
209.     public void onResults(Bundle results) {
210.         checkForCommands(results);
211.     }
212.  
213.     @Override
214.     public void onPartialResults(Bundle partialResults) {
215.         //checkForCommands(partialResults);
216.     }
217.  
218.     @Override
219.     public void onEvent(int eventType, Bundle params) {
220.  
221.     }
222.  
223.  
224. }

 

I finally did it, putting the m_EngineSR.startListening(voiceIntent); in the onStartCommand(), called each time we call the service with activity.startService(intent) , which is not the case of the OnCreate() function in the service.

Here is the final working code:

SpeechService.Java class, in custom Android Project

Code (CSharp):

1. package com.snuffchan.fps;
2.  
3. import android.app.Service;
4. import android.content.Context;
5. import android.content.Intent;
6. import android.os.Bundle;
7. import android.os.IBinder;
8. import android.speech.RecognitionListener;
9. import android.speech.RecognitionService;
10. import android.speech.RecognizerIntent;
11. import android.speech.SpeechRecognizer;
12. import android.util.Log;
13.  
14. import com.google.unity.GoogleUnityActivity;
15. import com.unity3d.player.UnityPlayer;
16.  
17. import java.util.ArrayList;
18.  
19. import static android.speech.SpeechRecognizer.ERROR_AUDIO;
20. import static android.speech.SpeechRecognizer.ERROR_CLIENT;
21. import static android.speech.SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS;
22. import static android.speech.SpeechRecognizer.ERROR_NETWORK;
23. import static android.speech.SpeechRecognizer.ERROR_NETWORK_TIMEOUT;
24. import static android.speech.SpeechRecognizer.ERROR_NO_MATCH;
25. import static android.speech.SpeechRecognizer.ERROR_RECOGNIZER_BUSY;
26. import static android.speech.SpeechRecognizer.ERROR_SERVER;
27. import static android.speech.SpeechRecognizer.ERROR_SPEECH_TIMEOUT;
28. import static android.speech.SpeechRecognizer.createSpeechRecognizer;
29.  
30. /**
31. * Created by alexandre on 16/02/16.
32. */
33. public class SpeechService extends RecognitionService implements RecognitionListener {
34.  
35.     public static SpeechRecognizer m_EngineSR;
36.     public static Context mContext;
37.     public static SpeechService mSpeechService;
38.     //public static android.app.Activity mActivity;
39.     static String TAG = "VOICE RECOGNITION";
40.  
41.     /**
42.      * StartListenning**************************************************
43.      * Static function call by the c# to launch the service SpeechService
44.      */
45.     public static void StartListenning(GoogleUnityActivity activity) {
46.         Log.i(TAG, "START THE SERVICE! ");
47.  
48.         if( UnityPlayer.currentActivity != null) {
49.  
50.                 UnityPlayer.UnitySendMessage("CubeMessage", "GetMessage", "TRY 2 RESTART SERVICE");
51.                 Intent intent = new Intent(activity, SpeechService.class);
52.                 activity.startService(intent);
53.            // }
54.         }
55.  
56.     }
57.  
58.     //The service is created and voice recognition service has been started
59.     @Override
60.     public void onCreate() {
61.  
62.         Log.i(TAG, "onCreate()");
63.         if(mSpeechService==null ) {
64.             mContext = getApplicationContext();
65.             mSpeechService = this;
66.             m_EngineSR = createSpeechRecognizer(this);
67.             m_EngineSR.setRecognitionListener(this);
68.             Intent voiceIntent = RecognizerIntent.getVoiceDetailsIntent(mContext);
69.             m_EngineSR.startListening(voiceIntent);
70.         }
71.         super.onCreate();
72.     }
73.  
74.     private void checkForCommands(Bundle bundle) {
75.         ArrayList<String> voiceText = bundle.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION);
76.         if (voiceText != null) {
77.             if (voiceText.size() > 0) {
78.                 SendMessageToUnity(voiceText.get(0));
79.             } else {
80.                 SendMessageToUnity("nothing!");
81.             }
82.  
83.         } else {
84.             SendMessageToUnity("voiceText empty!");
85.         }
86.     }
87.  
88.     /**
89.      *  Triggered by the activity.startService(intent);
90.      * @param intent intent send
91.      * @param flags flags
92.      * @param startId startid
93.      * @return
94.      */
95.     @Override
96.     public int onStartCommand(Intent intent, int flags, int startId){
97.         SendMessageToUnity("onStartCommand()!");
98.         if(mSpeechService!=null ) {
99.             mContext = getApplicationContext();
100.             restartListenning(mContext);
101.         }
102.  
103.         return Service.START_NOT_STICKY;
104.     }
105.  
106.     public static void restartListenning(Context context_){
107.  
108.         if(mSpeechService!=null) {
109.             mContext = context_;
110.             if (m_EngineSR == null){
111.                 m_EngineSR = createSpeechRecognizer(mSpeechService);
112.                 m_EngineSR.setRecognitionListener(mSpeechService);
113.             }else{
114.                 m_EngineSR.stopListening();
115.             }
116.             Intent voiceIntent = RecognizerIntent.getVoiceDetailsIntent(mContext);
117.             m_EngineSR.startListening(voiceIntent);
118.         }
119.     }
120.  
121.  
122.     /**
123.      * Send the data to Unity
124.      * @param text text  string
125.      */
126.     public void SendMessageToUnity(String text){
127.         if(m_EngineSR!=null) {
128.  
129.             if(text != null && text.isEmpty() )Log.i("TESTING: ", "final message! =" + text);
130.             try {
131.  
132.                 if( UnityPlayer.currentActivity != null){
133.                     UnityPlayer.UnitySendMessage("CubeMessage", "GetMessage", text);
134.                     SendStopWritingUnity();
135.                 }
136.  
137.             } catch (Exception e) {
138.                 // Log.e(TAG, "UnitySendMessage failed" + e.getMessage());
139.             }
140.             // m_EngineSR.stopListening();
141.         }
142.     }
143.  
144.     /**
145.      * Send an error to Unity
146.      * @param text errors string
147.      */
148.     public void SendErrorToUnity(String text){
149.         if(m_EngineSR!=null) {
150.             try {
151.  
152.                 if( UnityPlayer.currentActivity != null) UnityPlayer.UnitySendMessage("CubeMessage", "GetError", text);
153.  
154.             } catch (Exception e) {
155.                 // Log.e(TAG, "UnitySendMessage failed" + e.getMessage());
156.             }
157.             // m_EngineSR.stopListening();
158.         }
159.     }
160.  
161.     /**
162.      * We notifiy to unity than the player as stopped writing (link to the stop listenning funciton)
163.      */
164.     public void SendStopWritingUnity(){
165.         if(m_EngineSR!=null) {
166.             try {
167.  
168.                 if( UnityPlayer.currentActivity != null){
169.                     UnityPlayer.UnitySendMessage("CubeMessage", "StopWriting","");
170.  
171.                 }
172.  
173.             } catch (Exception e) {
174.                 // Log.e(TAG, "UnitySendMessage failed" + e.getMessage());
175.             }
176.             // m_EngineSR.stopListening();
177.         }
178.     }
179.  
180.  
181.     /***
182.      * SERVICE STUFF
183.      */
184.  
185.  
186.     @Override
187.     public void onDestroy() {
188.         if(m_EngineSR!= null)m_EngineSR.cancel();
189.         Log.i("SimpleVoiceService", "Service stopped");
190.         super.onDestroy();
191.     }
192.  
193.  
194.  
195.     @Override
196.     protected void onStartListening(Intent recognizerIntent, Callback listener) {
197.  
198.     }
199.  
200.     @Override
201.     protected void onCancel(Callback listener) {
202.  
203.     }
204.  
205.     @Override
206.     protected void onStopListening(Callback listener) {
207.         SendStopWritingUnity();
208.     }
209.  
210.  
211.     @Override
212.     public void onReadyForSpeech(Bundle params) {
213.  
214.     }
215.  
216.     @Override
217.     public void onBeginningOfSpeech() {
218.  
219.     }
220.  
221.     @Override
222.     public void onRmsChanged(float rmsdB){
223.     }
224.  
225.     @Override
226.     public void onBufferReceived(byte[] buffer) {
227.  
228.     }
229.  
230.     @Override
231.     public void onEndOfSpeech() {
232.  
233.     }
234.  
235.     @Override
236.     public void onError(int error) {
237.         try {
238.  
239.  
240.             String message;
241.             switch (error)
242.             {
243.                 case ERROR_AUDIO:
244.                     message = "Audio recording error";
245.                     break;
246.                 case ERROR_CLIENT:
247.                     message = "Client side error";
248.                     break;
249.                 case ERROR_INSUFFICIENT_PERMISSIONS:
250.                     message = "Insufficient permissions";
251.                     break;
252.                 case ERROR_NETWORK:
253.                     message = "Network error";
254.                     break;
255.                 case ERROR_NETWORK_TIMEOUT:
256.                     message = "Network timeout";
257.                     break;
258.                 case ERROR_NO_MATCH:
259.                     message = "No match";
260.                     break;
261.                 case ERROR_RECOGNIZER_BUSY:
262.                     message = "RecognitionService busy";
263.                     break;
264.                 case ERROR_SERVER:
265.                     message = "error from server";
266.                     break;
267.                 case ERROR_SPEECH_TIMEOUT:
268.                     message = "No speech input";
269.                     break;
270.                 default:
271.                     message = "Didn't understand, please try again.";
272.                     break;
273.             }
274.  
275.             SendErrorToUnity(message);
276.  
277.         } catch (Exception e) {
278.             e.printStackTrace();
279.         }
280.         //m_SRListener.onError(error);
281.     }
282.  
283.     @Override
284.     public void onResults(Bundle results) {
285.         checkForCommands(results);
286.     }
287.  
288.     @Override
289.     public void onPartialResults(Bundle partialResults) {
290.         //checkForCommands(partialResults);
291.     }
292.  
293.     @Override
294.     public void onEvent(int eventType, Bundle params) {
295.  
296.     }
297.  
298.  
299. }

The C# Script triggering speech receiving it, and handling errors.

Code (CSharp):

1. using UnityEngine;
2. using System.Collections;
3.  
4.  
5. /*************************************
6. *    MESSAGE MAKER
7. **************************************/
8. using System.Net;
9. using System.IO;
10.  
11.  
12. public class MessageBox : MonoBehaviour {
13.  
14.     public TextMesh textMesh;
15.     public TextMesh textErrorMesh;
16.  
17.     //Collider used as button
18.     public Collider colliderWrite;
19.     public Collider colliderPost;
20.  
21.     public CardboardHead head;
22.     public float timeBetween = 8f;
23.     public float timeCountdown = 0;
24.  
25.     public string URLServer = "http://xxxxxx.com/";
26.  
27.     public string userName ="Anon";
28.     public string userMessage ="";
29.     public string errorMessage ="";
30.  
31.    
32.     private bool isRecording = false;
33.    
34.     void Start(){
35.         //colliderWrite = GetComponent<Collider> ();
36.         //colliderPost = GetComponent<Collider> ();
37.         head = (CardboardHead) FindObjectOfType(typeof(CardboardHead));
38.     }
39.    
40.     void Update () {
41.         if (head != null) {
42.             bool isLooked;
43.             RaycastHit hit;
44.  
45.             if (colliderWrite != null) {
46.            
47.                 isLooked = colliderWrite.Raycast (head.Gaze, out hit, Mathf.Infinity);
48.                 if (isLooked) {
49.                     if (!isRecording && timeCountdown < 0) {
50.                         StartMessage ();
51.  
52.                     }
53.                 }
54.             }
55.  
56.             if (colliderPost != null) {
57.                 isLooked = colliderPost.Raycast (head.Gaze, out hit, Mathf.Infinity);
58.                 if (isLooked) SendMessageOnline();
59.  
60.             }
61.         }
62.         if (!isRecording) timeCountdown -= Time.deltaTime;
63.         GetComponent<Renderer>().material.color = isRecording ? Color.green : Color.red;
64.  
65.         if(textMesh != null) textMesh.text = userMessage;
66.     }
67.  
68.     /********************
69.      * EXTERNAL INTERFACE
70.      ********************/
71.  
72.     void StartMessage(){
73.  
74.  
75.         isRecording = true;
76.         timeCountdown  = timeBetween;
77.  
78.              AndroidJavaClass androidJC = new AndroidJavaClass("com.unity3d.player.UnityPlayer");
79.              AndroidJavaObject jo = androidJC.GetStatic<AndroidJavaObject>("currentActivity");
80.             AndroidJavaClass jc = new AndroidJavaClass("com.snuffchan.fps.SpeechService");
81.              jc.CallStatic("StartListenning", jo);
82.            
83.  
84.     }
85.    
86.     void GetMessage(string message_){
87.         userMessage = message_;
88.         if(textMesh != null) textMesh.text = message_.ToUpper();
89.  
90.     }
91.  
92.     void GetError(string message_){
93.         errorMessage = message_;
94.         if(textErrorMesh != null) textErrorMesh.text = message_;
95.     }
96.  
97.     void StopWriting(string message_){
98.         isRecording = false;
99.     }
100.  
101.     /********************
102.      * POSTING ONLINE FUNCTIONS
103.      ********************/
104.    
105.     void SendMessageOnline(){
106.  
107.         if (userMessage.Length > 0) {
108.  
109.             WWWForm form = new WWWForm ();
110.             form.AddField ("pseudo", userName);
111.             form.AddField ("texte", userMessage);
112.             userMessage = "";
113.  
114.             WWW www = new WWW (URLServer + "php/chanWriter.php", form);
115.             StartCoroutine (WaitForRequest (www));
116.             }
117.  
118.         }
119.  
120.         public IEnumerator WaitForRequest(WWW www){
121.         yield return www;
122.                 // check for errors
123.                 if (www.error == null)
124.                 {
125.                     Debug.Log("WWW Ok!: " + www.data);
126.                 } else {
127.                     Debug.Log("WWW Error: "+ www.error);
128.                 }  
129.         }  
130.  
131. }
132.