I am trying to develop a Human Activity Recognition application using Xamarin. I have developed a TensorFlow model and converted that to Tensorflow Lite and able to load the model to the app but not sure how to load the input data.
Input data: Accelerometer and Gyroscope sensor data from the phone. shape: 128*9.
I Have SensorModel class as bellow
public class SensorModel
{
public double Acc_X { get; set; }
public double Acc_Y { get; set; }
public double Acc_Z { get; set; }
public double Gyro_X { get; set; }
public double Gyro_Y { get; set; }
public double Gyro_Z { get; set; }
public double Acc_Total_X { get; set; }
public double Acc_Total_Y { get; set; }
public double Acc_Total_Z { get; set; }
}
I have my Classifier method as below:
public List<ActivityModel> Classify(List<SensorModel> sensorData)
{
var assetDescriptor = Application.Context.Assets.OpenFd("converted_model.tflite");
var inputStream = new FileInputStream(assetDescriptor.FileDescriptor);
var mappedByteBuffer = inputStream.Channel.Map(FileChannel.MapMode.ReadOnly, assetDescriptor.StartOffset, assetDescriptor.DeclaredLength);
var interpreter = new Xamarin.TensorFlow.Lite.Interpreter(mappedByteBuffer);
//sensor data to byte array
BinaryFormatter bf = new BinaryFormatter();
MemoryStream ms = new MemoryStream();
bf.Serialize(ms, sensorData);
var input = ms.ToArray();
//var sr = new StreamReader(Application.Context.Assets.Open("activity_labels.txt"));
//var labels = sr.ReadToEnd().Split('\n').Select(s => s.Trim()).Where(s => !string.IsNullOrEmpty(s)).ToList();
var labels = new List<string> { "1", "2", "3", "4", "5", "6" };
var outputLocations = new float[labels.Count];
var output = Java.Lang.Object.FromArray(outputLocations);
try
{
interpreter.Run(input, output);
}
catch (Exception ex)
{
throw new Exception(ex.ToString());
}
var classificationResult = output.ToArray<float[]>();
////Map the classificationResult to the labels and sort the result to find which label has the highest probability
var classificationModelList = new List<ActivityModel>();
for (var i = 0; i < labels.Count; i++)
{
var label = labels[i]; classificationModelList.Add(new ActivityModel(label, classificationResult[0][i]));
}
return classificationModelList;
}
My input data should be of size 128*9. It's a list of sensor data. (type float or double).
But getting this Exception.
{Java.Lang.IllegalArgumentException: Cannot convert between a TensorFlowLite tensor with type FLOAT32 and a Java object of type [B (which is compatible with the TensorFlowLite type UINT8).
at Java.Interop.JniEnvironment+InstanceMethods.CallVoidMethod (Java.Interop.JniObjectReference instance, Java.Interop.JniMethodInfo method, Java.Interop.JniArgumentValue* args) [0x0006e] in <dac4c5a4b77f4e61a5e6d9d3050dfb9f>:0
at Java.Interop.JniPeerMembers+JniInstanceMethods.InvokeAbstractVoidMethod (System.String encodedMember, Java.Interop.IJavaPeerable self, Java.Interop.JniArgumentValue* parameters) [0x00014] in <dac4c5a4b77f4e61a5e6d9d3050dfb9f>:0
at Xamarin.TensorFlow.Lite.Interpreter.Run (Java.Lang.Object input, Java.Lang.Object output) [0x00053] in <4ff04480b90243d7b71dc70e02d1dbfb>:0
at HAR_Test_2.ActivityClassifier.Classify (System.Collections.Generic.List`1[T] sensorData) [0x000cf] in C:\Users\Nandan\Documents\Visual Studio 2019\Projects\HAR_Test_2\HAR_Test_2\ActivityClassifier.cs:47
--- End of managed Java.Lang.IllegalArgumentException stack trace ---
java.lang.IllegalArgumentException: Cannot convert between a TensorFlowLite tensor with type FLOAT32 and a Java object of type [B (which is compatible with the TensorFlowLite type UINT8).
at org.tensorflow.lite.Tensor.throwIfTypeIsIncompatible(Tensor.java:316)
at org.tensorflow.lite.Tensor.getInputShapeIfDifferent(Tensor.java:218)
at org.tensorflow.lite.NativeInterpreterWrapper.run(NativeInterpreterWrapper.java:137)
at org.tensorflow.lite.Interpreter.runForMultipleInputsOutputs(Interpreter.java:311)
at org.tensorflow.lite.Interpreter.run(Interpreter.java:272)
at crc6455763083a08ecf13.MainActivity.n_onSensorChanged(Native Method)
at crc6455763083a08ecf13.MainActivity.onSensorChanged(MainActivity.java:75)
at android.hardware.SystemSensorManager$SensorEventQueue.dispatchSensorEvent(SystemSensorManager.java:833)
at android.os.MessageQueue.nativePollOnce(Native Method)
at android.os.MessageQueue.next(MessageQueue.java:326)
at android.os.Looper.loop(Looper.java:160)
at android.app.ActivityThread.main(ActivityThread.java:6669)
at java.lang.reflect.Method.invoke(Native Method)
at com.android.internal.os.RuntimeInit$MethodAndArgsCaller.run(RuntimeInit.java:493)
at com.android.internal.os.ZygoteInit.main(ZygoteInit.java:858)
}
base: {Java.Lang.RuntimeException}
JniPeerMembers: {Android.Runtime.XAPeerMembers}
Can anyone suggest me how I am supposed to feed the data to the interpreter?
Solved the problem. I had to feed the interpreter with Array of float[] but I was passing List of SensorModel. Now I converted List of SensorModel into float[] which is working like a charm. I am adding the code here
public List<ActivityModel> Classify(List<SensorModel> sensorData)
{
var assetDescriptor = Application.Context.Assets.OpenFd("converted_model.tflite");
var inputStream = new FileInputStream(assetDescriptor.FileDescriptor);
var mappedByteBuffer = inputStream.Channel.Map(FileChannel.MapMode.ReadOnly, assetDescriptor.StartOffset, assetDescriptor.DeclaredLength);
var interpreter = new Xamarin.TensorFlow.Lite.Interpreter(mappedByteBuffer);
var InputArray = PrepareInputArray(sensorData).ToArray();
var Input_tensor = InputArray.SelectMany(x => x).ToArray();
var sr = new StreamReader(Application.Context.Assets.Open("activity_labels.txt"));
var labels = sr.ReadToEnd().Split('\n').Select(s => s.Trim()).Where(s => !string.IsNullOrEmpty(s)).ToList();
var outputLocations = new float[1][] { new float[labels.Count] };
var outputs = Java.Lang.Object.FromArray(outputLocations);
try
{
interpreter.Run(Input_tensor, outputs);
}
catch (Exception ex)
{
throw new Exception(ex.ToString());
}
var classificationResult = outputs.ToArray<float[]>();
////Map the classificationResult to the labels and sort the result to find which label has the highest probability
var classificationModelList = new List<ActivityModel>();
for (var i = 0; i < labels.Count; i++)
{
var label = labels[i]; classificationModelList.Add(new ActivityModel(label, classificationResult[0][i]));
}
return classificationModelList;
}
private List<List<float>> PrepareInputArray(List<SensorModel> sensorData)
{
List<List<float>> record = new List<List<float>>();
sensorData.ForEach(el =>
{
record.Add(new List<float> { el.Acc_X, el.Acc_Y, el.Acc_Z, el.Gyro_X, el.Gyro_Y, el.Gyro_Z, el.Acc_Total_X, el.Acc_Total_Y, el.Acc_Total_Z });
});
return record;
}
Thank you friends.