I am using pocketsphinx in a Xamarin Android app (using MVVMCross) for speech recognition. It works great when I use it in the View, and it works great when I moved it from the View into a plugin so that I could use it in the ViewModel (using callback event handlers).
Now I am refactoring (again!) to use async/await, and the app fails to load the pocketsphinx library. My code is below. When I reach the call
Java.Lang.JavaSystem.LoadLibrary("pocketsphinx_jni");
I get an exception:
Java.Lang.UnsatisfiedLinkError: Library pocketsphinx_jni not found; tried [/vendor/lib/libpocketsphinx_jni.so, /system/lib/libpocket...
I've tried a combination of things using Application.Context.ApplicationInfo.NativeLibraryDir and Java.Lang.JavaSystem.Load but no joy.
Any ideas on why this would start to happen now? Presumably something to do with threading, but I am stumped as to how to solve the issue. All of this code worked fine before I started refactoring to use async/await.
The call in my ViewModel is:
var result = await _speechRecognitionService.ListenAsync(delay);
And here is my SpeechRecognitionService class:
public class SpeechRecognitionService : Java.Lang.Object,
ISpeechRecognitionService, IRecognitionListener
{
private SpeechRecognizer _recognizer;
private Java.IO.File _assetDir;
private bool _isHandlingResponse;
private bool _isInitialised;
private static EventWaitHandle _waitHandle = new AutoResetEvent(false);
private string _response;
public event EventHandler HeardSpeechEventHandler;
public bool IsInitialised()
{
return _isInitialised;
}
/// <summary>
/// Hopefully fire off the listen asynchronously
/// </summary>
/// <param name="delay"></param>
/// <returns></returns>
public async Task<string> ListenAsync(int delay)
{
return await Task.Run(() => DoListen(delay));
}
/// <summary>
/// Initialise if required then fire up the listener
/// </summary>
/// <param name="delay"></param>
/// <returns></returns>
private string DoListen(int delay)
{
if (!IsInitialised())
{
Init();
}
Listen(delay);
return "";
}
/// <summary>
/// Set up the config for the listener
/// </summary>
public void Init()
{
Console.WriteLine("SpeechRecognitionIntentService ====== Init");
try
{
// THIS CALL FAILS!
Java.Lang.JavaSystem.LoadLibrary("pocketsphinx_jni");
_assetDir = Edu.Cmu.Pocketsphinx.Assets.SyncAssets(Application.Context);
}
catch (IOException e)
{
throw new Exception("Failed to synchronize assets when attempting to load voice recognition software", e);
}
var config = Decoder.DefaultConfig();
config.SetString("-dict", JoinPath(_assetDir, "models/lm/cmu07a.dic"));
config.SetString("-hmm", JoinPath(_assetDir, "models/hmm/en-us-semi"));
config.SetString("-rawlogdir", _assetDir.AbsolutePath);
config.SetInt("-maxhmmpf", 10000);
config.SetBoolean("-fwdflat", false);
config.SetBoolean("-bestpath", false);
config.SetFloat("-kws_threshold", 1e-5);
//config.SetFloat("-samprate", 8000);
//config.SetBoolean("-remove_noise", false);
_recognizer = new SpeechRecognizer(config);
_recognizer.AddListener(this);
// Create keyword-activation search.
//_recognizer.SetKws(KwsSearchName, Keyphrase);
// Create grammar-based search for guide.
var lw = config.GetInt("-lw");
AddSearch("guide", "guide.gram", "<guide.command>", lw);
// Create language model search.
//var path = JoinPath(_assetDir, "models/lm/weather.dmp");
//var lm = new NGramModel(config, _recognizer.Logmath, path);
//_recognizer.SetLm("forecast", lm);
_isInitialised = true;
Console.WriteLine("InitVoiceRecognition done.");
}
/// <summary>
/// Listen for "delay" seconds. Use a timer. The timer and the speech recognition callbacks both use the _isHandlingResponse flag to decide
/// who is handling the response. If the user speaks before the timer elapses then the speech recognizer callbacks set the flag and handle
/// it by returning the spoken string to the HeardSpeechEventHandler. If the timer elapses first, then it sets the flag and sends an empty string
/// back to the HeardSpeechEventHandler.
/// </summary>
/// <param name="delay"></param>
public string Listen(int delay)
{
Console.WriteLine("Start listening for {0} seconds.", delay);
//_recognizer.StopListening();
try
{
_isHandlingResponse = false;
_recognizer.SetSearch("guide");
var timer = new Timer(_ =>
{
// If the speech recognizer is not handling the response already (i.e. the user hasn't spoken) then
// set the flag and return an empty response
if (!_isHandlingResponse)
{
_isHandlingResponse = true;
_recognizer.StopListening();
//Debug.WriteLine("The timer elapsed for the delay of {0} seconds with NO RESPONSE ({0})", delay, CurrentObject.Label);
//if (HeardSpeechEventHandler != null)
//{
// HeardSpeechEventHandler(this, new SpeechResult() {Result = string.Empty});
//}
Console.Write("Timer elapsed so setting response to '' and setting wait handle");
_response = string.Empty;
_waitHandle.Set();
}
}, null, delay * 1000, Timeout.Infinite);
_recognizer.StartListening();
Console.WriteLine("Started listening, now waiting for wait handle...");
_waitHandle.WaitOne();
return _response;
}
catch (Exception ex)
{
Console.WriteLine("An exception occurred attempting to listen: {0}", ex.Message);
return "";
}
}
//public IntPtr Handle { get; private set; }
/// <summary>
///
/// </summary>
/// <param name="p0"></param>
public void OnPartialResult(Hypothesis p0)
{
Console.WriteLine("Partial result: " + p0.Hypstr);
// If the timer is not handling the response already (i.e. timer hasn't timed out yet) then
// set the flag and stop listening. This will kick off the OnResult function.
if (!_isHandlingResponse)
{
_isHandlingResponse = true;
_recognizer.StopListening();
Console.WriteLine("Partial result: " + p0.Hypstr);
}
}
//http://www.albahari.com/
/// <summary>
///
/// </summary>
/// <param name="p0"></param>
public void OnResult(Hypothesis p0)
{
//if (!_isHandlingResponse)
//{
// _isHandlingResponse = true;
Console.WriteLine("Full result, setting response to '{0}' and setting wait handle", p0.Hypstr);
//if (HeardSpeechEventHandler != null)
//{
// HeardSpeechEventHandler(this, new SpeechResult() {Result = p0.Hypstr});
//}
_response = p0.Hypstr;
_waitHandle.Set();
//}
}
public void OnVadStateChanged(bool p0)
{
//throw new NotImplementedException();
//Console.WriteLine("OnVadStateChanged: {0}", p0);
}
/// <summary>
///
/// </summary>
/// <param name="name"></param>
/// <param name="path"></param>
/// <param name="ruleName"></param>
/// <param name="lw"></param>
private void AddSearch(String name, String path, String ruleName, int lw)
{
//File grammarParent = new File(Path.Combine(_assetDir.AbsolutePath, "grammar"));
var grammarParent = JoinPath(_assetDir, "grammar");
var jsgf = new Jsgf(Path.Combine(grammarParent, path));
var rule = jsgf.GetRule(ruleName);
var fsg = jsgf.BuildFsg(rule, _recognizer.Logmath, lw);
_recognizer.SetFsg(name, fsg);
Console.WriteLine("AddSearch done.");
}
/// <summary>
///
/// </summary>
/// <param name="parent"></param>
/// <param name="path"></param>
/// <returns></returns>
private static String JoinPath(Java.IO.File parent, String path)
{
return new Java.IO.File(parent, path).AbsolutePath;
}
}
I got this to work by getting the UI thread of the current activity to load the Pocket Sphinx library, like so:
Mvx.Resolve<IMvxAndroidCurrentTopActivity>().Activity.RunOnUiThread(() =>
Java.Lang.JavaSystem.LoadLibrary("pocketsphinx_jni"));
It feels a little wrong to do it like this (why should my plugin need to know anything about the current UI thread?) - but it works.