I recently checked out PocketSphinx powered by CMUSphinx for Android found here: https://github.com/cmusphinx/pocketsphinx-android-demo. I've modified it a little bit to play around and see what I could do.
public class PocketSphinxActivity extends Activity implements
RecognitionListener {
/* Named searches allow to quickly reconfigure the decoder */
private static final String KWS_SEARCH = "wakeup";
private static final String OVER_COMMAND = "overshield";
private static final String INVISIBLE_COMMAND = "invis";
private static final String POWER_COMMAND = "power";
private static final String MENU_SEARCH = "menu";
/* Keyword we are looking for to activate menu */
private static final String KEYPHRASE = "ready coach";
private boolean isGameStarted = false;
/* Used to handle permission request */
private static final int PERMISSIONS_REQUEST_RECORD_AUDIO = 1;
private SpeechRecognizer recognizer;
private HashMap<String, Integer> captions;
@Override
public void onCreate(Bundle state) {
super.onCreate(state);
// Prepare the data for UI
captions = new HashMap<String, Integer>();
captions.put(KWS_SEARCH, R.string.kws_caption);
captions.put(MENU_SEARCH, R.string.menu_caption);
captions.put(INVISIBLE_COMMAND, R.string.invis_caption);
captions.put(OVER_COMMAND, R.string.os_caption);
setContentView(R.layout.main);
((TextView) findViewById(R.id.caption_text))
.setText("Preparing the recognizer");
// Check if user has given permission to record audio
int permissionCheck = ContextCompat.checkSelfPermission(getApplicationContext(), Manifest.permission.RECORD_AUDIO);
if (permissionCheck == PackageManager.PERMISSION_DENIED) {
ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.RECORD_AUDIO}, PERMISSIONS_REQUEST_RECORD_AUDIO);
return;
}
runRecognizerSetup();
}
private void runRecognizerSetup() {
// Recognizer initialization is a time-consuming and it involves IO,
// so we execute it in async task
new AsyncTask<Void, Void, Exception>() {
@Override
protected Exception doInBackground(Void... params) {
try {
Assets assets = new Assets(PocketSphinxActivity.this);
File assetDir = assets.syncAssets();
setupRecognizer(assetDir);
} catch (IOException e) {
return e;
}
return null;
}
@Override
protected void onPostExecute(Exception result) {
if (result != null) {
((TextView) findViewById(R.id.caption_text))
.setText("Failed to init recognizer " + result);
} else {
switchSearch(KWS_SEARCH);
}
}
}.execute();
}
@Override
public void onRequestPermissionsResult(int requestCode,
String[] permissions, int[] grantResults) {
super.onRequestPermissionsResult(requestCode, permissions, grantResults);
if (requestCode == PERMISSIONS_REQUEST_RECORD_AUDIO) {
if (grantResults.length > 0 && grantResults[0] == PackageManager.PERMISSION_GRANTED) {
runRecognizerSetup();
} else {
finish();
}
}
}
@Override
public void onDestroy() {
super.onDestroy();
if (recognizer != null) {
recognizer.cancel();
recognizer.shutdown();
}
}
/**
* In partial result we get quick updates about current hypothesis. In
* keyword spotting mode we can react here, in other modes we need to wait
* for final result in onResult.
*/
@Override
public void onPartialResult(Hypothesis hypothesis) {
if (hypothesis == null)
return;
String text = hypothesis.getHypstr();
if (text.equals(KEYPHRASE))
switchSearch(MENU_SEARCH);
//Invisible Mentioned
else if (text.equals(INVISIBLE_COMMAND))
switchSearch(INVISIBLE_COMMAND);
//Over Mentioned
else if (text.equals(OVER_COMMAND))
switchSearch(OVER_COMMAND);
else if (text.equals(POWER_COMMAND))
((TextView) findViewById(R.id.result_text)).setText(text);
else
((TextView) findViewById(R.id.result_text)).setText(text);
}
/**
* This callback is called when we stop the recognizer.
*/
@Override
public void onResult(Hypothesis hypothesis) {
((TextView) findViewById(R.id.result_text)).setText("");
if (hypothesis != null) {
String text = hypothesis.getHypstr();
makeText(getApplicationContext(), text, Toast.LENGTH_SHORT).show();
}
}
@Override
public void onBeginningOfSpeech() {
}
/**
* We stop recognizer here to get a final result
*/
@Override
public void onEndOfSpeech() {
if (!recognizer.getSearchName().equals(KWS_SEARCH))
switchSearch(KWS_SEARCH);
}
private void switchSearch(String searchName) {
recognizer.stop();
// If we are not spotting, start listening with timeout (10000 ms or 10 seconds).
if (searchName.equals(KEYPHRASE)) {
isGameStarted = true;
startTimer();
} else {
if (isGameStarted)
recognizer.startListening(searchName);
else
recognizer.startListening(searchName, 10000);
String caption = getResources().getString(captions.get(searchName));
((TextView) findViewById(R.id.caption_text)).setText(caption);
}
}
private void setupRecognizer(File assetsDir) throws IOException {
// The recognizer can be configured to perform multiple searches
// of different kind and switch between them
recognizer = SpeechRecognizerSetup.defaultSetup()
.setAcousticModel(new File(assetsDir, "en-us-ptm"))
.setDictionary(new File(assetsDir, "cmudict-en-us.dict"))
// .setRawLogDir(assetsDir) // To disable logging of raw audio comment out this call (takes a lot of space on the device)
.getRecognizer();
recognizer.addListener(this);
// Create keyword-activation search.
recognizer.addKeyphraseSearch(KWS_SEARCH, KEYPHRASE);
// Create grammar-based search for selection between demos
File menuGrammar = new File(assetsDir, "menu.gram");
recognizer.addGrammarSearch(MENU_SEARCH, menuGrammar);
}
@Override
public void onError(Exception error) {
((TextView) findViewById(R.id.caption_text)).setText(error.getMessage());
}
@Override
public void onTimeout() {
switchSearch(KWS_SEARCH);
}
public void startTimer() {
new CountDownTimer(30000, 1000) {
public void onTick(long millisUntilFinished) {
((TextView) findViewById(R.id.caption_text))
.setText("seconds remaining: " + millisUntilFinished / 1000);
}
public void onFinish() {
isGameStarted = false;
// mTextField.setText("done!");
}
}.start();
}
}
Everything works fine when I load it up but I have been experimenting and trying to change the words that it recognizes. It seems to recognize "stop" fine but the other ones are giving me problems. Right now I have the following Grammer setup:
#JSGF V1.0;
grammar menu;
public = (invis | invisible | overshield | stop | power | close);
I've also added the custom words to my dictionary file just to make sure they were in there.
Here is the error log i'm getting, what else do I need to do to incorporate these words into my project?
11-22 23:17:02.812 6856-6856/edu.cmu.sphinx.pocketsphinx I/SpeechRecognizer: Stop recognition
11-22 23:17:02.812 6856-6856/edu.cmu.sphinx.pocketsphinx I/SpeechRecognizer: Start recognition "overshield"
11-22 23:17:02.812 6856-6856/edu.cmu.sphinx.pocketsphinx D/AndroidRuntime: Shutting down VM
11-22 23:17:02.812 6856-6856/edu.cmu.sphinx.pocketsphinx E/AndroidRuntime: FATAL EXCEPTION: main
Process: edu.cmu.sphinx.pocketsphinx, PID: 6856
java.lang.RuntimeException: Decoder_setSearch returned -1
at edu.cmu.pocketsphinx.PocketSphinxJNI.Decoder_setSearch(Native Method)
at edu.cmu.pocketsphinx.Decoder.setSearch(Decoder.java:181)
at edu.cmu.pocketsphinx.SpeechRecognizer.startListening(SpeechRecognizer.java:141)
at edu.cmu.pocketsphinx.demo.PocketSphinxActivity.switchSearch(PocketSphinxActivity.java:215)
at edu.cmu.pocketsphinx.demo.PocketSphinxActivity.onPartialResult(PocketSphinxActivity.java:170)
at edu.cmu.pocketsphinx.SpeechRecognizer$ResultEvent.execute(SpeechRecognizer.java:427)
at edu.cmu.pocketsphinx.SpeechRecognizer$RecognitionEvent.run(SpeechRecognizer.java:391)
at android.os.Handler.handleCallback(Handler.java:739)
at android.os.Handler.dispatchMessage(Handler.java:95)
at android.os.Looper.loop(Looper.java:158)
at android.app.ActivityThread.main(ActivityThread.java:7229)
at java.lang.reflect.Method.invoke(Native Method)
at com.android.internal.os.ZygoteInit$MethodAndArgsCaller.run(ZygoteInit.java:1230)
at com.android.internal.os.ZygoteInit.main(ZygoteInit.java:1120)
You confused "search" and a "command". You have added two searches - KWS_SEARCH and MENU_SEARCH, you can switch between them. MENU_SEARCH more properly should be called COMMAND_SEARCH. You can not switch to OVER_COMMAND because it is a command not a search and this is what exception tells you. You need to perform actions and switch to KWS_SEARCH instead.
Your code should look like this:
@Override
public void onPartialResult(Hypothesis hypothesis) {
if (hypothesis == null)
return;
String text = hypothesis.getHypstr();
if (text.equals(KEYPHRASE))
switchSearch(MENU_SEARCH);
}
@Override
public void onResult(Hypothesis hypothesis) {
if (hypothesis == null)
return;
String text = hypothesis.getHypstr();
if (text.equals(OVER_COMMAND)) {
// RUN ACTIONS FOR OVER_COMMAND_HERE
}
}
@Override
public void onBeginningOfSpeech() {
}
@Override
public void onEndOfSpeech() {
if (!recognizer.getSearchName().equals(KWS_SEARCH))
switchSearch(KWS_SEARCH);
}