I am creating an app for my thesis and so far I have managed to create an update-able image classifying model that I trained using Keras and converted using coremltools. (I will provide the code for training and converting the model as I think it has something to do with my problem as well as the code that I use to update the model in-app.) I can successfully classify an image and the model works as intended for that, but the updating part doesn't. After I perform an update with an image and try to classify the same image with the "updated" model I get the same classification. Not even a small difference in confidence.
Model generation and conversion is done in Python. App is written in Swift.
Here is the code for creating the model:
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense, BatchNormalization
from keras import backend as K
img_width, img_height = 224, 224
train_data_dir = 'data/train/'
validation_data_dir = 'data/validate/'
nb_train_samples = 500
nb_validation_samples = 40
epochs = 20
batch_size = 25
if K.image_data_format() == 'channels_first':
input_shape = (3, img_width, img_height)
else:
input_shape = (img_width, img_height, 3)
model = Sequential()
model.add(Conv2D(32, (2, 2), input_shape=input_shape))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(BatchNormalization())
model.add(Conv2D(64, (2, 2)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(BatchNormalization())
model.add(Flatten())
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(2))
model.add(Activation('softmax'))
model.summary()
model.compile(loss = 'categorical_crossentropy',
optimizer = 'sgd',
metrics = ['accuracy'])
train_datagen = ImageDataGenerator()
test_datagen = ImageDataGenerator()
train_generator = train_datagen.flow_from_directory(train_data_dir,
target_size =(img_width, img_height),
batch_size = batch_size, class_mode ='categorical')
validation_generator = test_datagen.flow_from_directory(
validation_data_dir,
target_size =(img_width, img_height),
batch_size = batch_size, class_mode ='categorical')
model.fit_generator(train_generator,
steps_per_epoch = nb_train_samples // batch_size,
epochs = epochs, validation_data = validation_generator,
validation_steps = nb_validation_samples // batch_size)
model.save('ModelsH5/catndog_small.h5')
Here is the code for converting it to CoreML and make it update-able:
import coremltools
from coremltools.models.neural_network import SgdParams
from coremltools.models import MLModel
model = "catndog_small"
coreml_model_path = 'ModelsML/' + model + '.mlmodel'
neuralnetwork_spec = coremltools.utils.load_spec(coreml_model_path)
builder = coremltools.models.neural_network.NeuralNetworkBuilder(spec=neuralnetwork_spec)
builder.inspect_layers(last=3)
builder.inspect_input_features()
neuralnetwork_spec.description.metadata.author = 'Ervins Balodis'
neuralnetwork_spec.description.metadata.license = 'No license'
neuralnetwork_spec.description.metadata.shortDescription = ('Cat and Dog Classifier converted from a Keras model')
model_spec = builder.spec
builder.make_updatable(['dense_2'])
builder.set_categorical_cross_entropy_loss(name='lossLayer', input='classLabelProbs')
from coremltools.models.neural_network import SgdParams
builder.set_sgd_optimizer(SgdParams(lr=0.1, batch=1))
builder.set_epochs(10)
model_spec.isUpdatable = True
model_spec.specificationVersion = coremltools._MINIMUM_UPDATABLE_SPEC_VERSION
model_spec.description.trainingInput[0].shortDescription = 'Image for training and updating the model'
model_spec.description.trainingInput[1].shortDescription = 'Set the value as Cat or Dog and update the model'
updateableModel = 'ModelsML/' + model + '_updateable' + '.mlmodel'
coremltools.utils.save_spec(model_spec, updateableModel)
Here is the code for updating the model from in-app:
func createTrainingData(imageArray: [UIImage], outputLabel: String) -> MLArrayBatchProvider{
var featureProviders = [MLFeatureProvider]()
let inputName = "image"
let outputName = outputLabel
var trainingImages = [UIImage]()
trainingImages.append(theImage!)
for image in trainingImages {
let inputValue = try? MLFeatureValue(cgImage: image.cgImage!,
pixelsWide: 224,
pixelsHigh: 224,
pixelFormatType: kCVPixelFormatType_OneComponent8,
options: nil)
let outputValue = MLFeatureValue(string: String(outputName))
let dataPointFeatures: [String: MLFeatureValue] = [inputName: inputValue!,
outputName: outputValue]
if let provider = try? MLDictionaryFeatureProvider(dictionary: dataPointFeatures) {
featureProviders.append(provider)
}
else{
print("Failed to append feature provider.")
}
}
return MLArrayBatchProvider(array: featureProviders)
}
func updateModel(at url: URL,
with trainingData: MLArrayBatchProvider,
completionHandler: @escaping (MLUpdateContext) -> Void) {
// Create an Update Task.
guard let updateTask = try? MLUpdateTask(forModelAt: url,
trainingData: trainingData,
configuration: nil,
completionHandler: completionHandler)
else {
print("Couldn't create an MLUpdateTask.")
return
}
updateTask.resume()
}
func performUpdate(){
//Might need to change from a batch to a single image
let updateImages: [UIImage] = [theImage!]
let imageBatch = createTrainingData(imageArray: updateImages, outputLabel: "dog") // temp outputLabel
updateModel(at: globalCompiledModel!, with: imageBatch, completionHandler:{ param in
print("Model updated!")
})
}
I'm suspicious about globalCompiledModel. After updating the model, you'll need to re-load it, i.e. create a new instance from the URL where you saved the updated model. I also don't see where in your code you're saving the updated model.
(OK, technically speaking you don't have to save the updated model, but you do need to use the instance from the MLUpdateTaskContext afterwards.)