I am following this tutorial to build a simple deep learning app for traffic sign recognition. link
I made an own model, and I also tried it with the model in this repository: link
When I run the app from xcode on my iPhone, I can see the picture of the camera, but the text always says "Label", no matter what is on the screen. The only thing I modified from the tutorial is that I hardcoded the classes before converting to mlmodel:
# import necessary packages
from keras.models import load_model
import coremltools
import argparse
import pickle
# construct the argument parser and parse the arguments
# load the class labels
print("[INFO] loading class labels from label binarizer")
# lb = pickle.loads(open(args["labelbin"], "rb").read())
# class_labels = lb.classes_.tolist()
class_labels = list(range(1, 43))
print("[INFO] class labels: {}".format(class_labels))
# load the trained convolutional neural network
print("[INFO] loading model...")
model = load_model('my_model.h5')
# convert the model to coreml format
print("[INFO] converting model")
coreml_model = coremltools.converters.keras.convert(model,
input_names="image",
image_input_names="image",
image_scale=1/255.0,
class_labels=class_labels,
is_bgr=True)
# save the model to disk
output = "mymodel.mlmodel"
print("[INFO] saving model as {}".format(output))
coreml_model.save(output)
So instead of using laber binarizer, I told the converter that there are 43 classes in my model.
Here is my AppDelegate.swift:
//
// AppDelegate.swift
// trafficsign
//
// Created by administrator on 2020. 11. 11..
// Copyright © 2020. administrator. All rights reserved.
//
import UIKit
@UIApplicationMain
class AppDelegate: UIResponder, UIApplicationDelegate {
var window: UIWindow?
func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool {
// Override point for customization after application launch.
// Override point for customization after application launch.
window = UIWindow()
window?.makeKeyAndVisible()
let vc = ViewController()
window?.rootViewController = vc
return true
}
}
My SceneDelegate.swift:
//
// SceneDelegate.swift
// trafficsign
//
// Created by administrator on 2020. 11. 11..
// Copyright © 2020. administrator. All rights reserved.
//
import UIKit
class SceneDelegate: UIResponder, UIWindowSceneDelegate {
var window: UIWindow?
func scene(_ scene: UIScene, willConnectTo session: UISceneSession, options connectionOptions: UIScene.ConnectionOptions) {
// Use this method to optionally configure and attach the UIWindow `window` to the provided UIWindowScene `scene`.
// If using a storyboard, the `window` property will automatically be initialized and attached to the scene.
// This delegate does not imply the connecting scene or session are new (see `application:configurationForConnectingSceneSession` instead).
guard let windowScene = (scene as? UIWindowScene) else { return }
window = UIWindow(windowScene: windowScene)
window?.rootViewController = ViewController()
window?.makeKeyAndVisible()
}
func sceneDidDisconnect(_ scene: UIScene) {
// Called as the scene is being released by the system.
// This occurs shortly after the scene enters the background, or when its session is discarded.
// Release any resources associated with this scene that can be re-created the next time the scene connects.
// The scene may re-connect later, as its session was not neccessarily discarded (see `application:didDiscardSceneSessions` instead).
}
func sceneDidBecomeActive(_ scene: UIScene) {
// Called when the scene has moved from an inactive state to an active state.
// Use this method to restart any tasks that were paused (or not yet started) when the scene was inactive.
}
func sceneWillResignActive(_ scene: UIScene) {
// Called when the scene will move from an active state to an inactive state.
// This may occur due to temporary interruptions (ex. an incoming phone call).
}
func sceneWillEnterForeground(_ scene: UIScene) {
// Called as the scene transitions from the background to the foreground.
// Use this method to undo the changes made on entering the background.
}
func sceneDidEnterBackground(_ scene: UIScene) {
// Called as the scene transitions from the foreground to the background.
// Use this method to save data, release shared resources, and store enough scene-specific state information
// to restore the scene back to its current state.
}
}
And most importantly my SceneDelegate.swift:
//
// ViewController.swift
// trafficsign
//
// Created by administrator on 2020. 11. 11..
// Copyright © 2020. administrator. All rights reserved.
//
import UIKit
import AVFoundation
import Vision
class ViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate {
let label: UILabel = {
let label = UILabel()
label.textColor = .white
label.translatesAutoresizingMaskIntoConstraints = false
label.text = "Label"
label.font = label.font.withSize(30)
return label
}()
override func viewDidLoad() {
super.viewDidLoad()
setupCaptureSession()
view.addSubview(label)
setupLabel()
}
override func didReceiveMemoryWarning() {
// call the parent function
super.didReceiveMemoryWarning()
// Dispose of any resources that can be recreated.
}
func setupCaptureSession() {
// create a new capture session
let captureSession = AVCaptureSession()
// find the available cameras
let availableDevices = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInWideAngleCamera], mediaType: AVMediaType.video, position: .back).devices
do {
// select a camera
if let captureDevice = availableDevices.first {
captureSession.addInput(try AVCaptureDeviceInput(device: captureDevice))
}
} catch {
// print an error if the camera is not available
print(error.localizedDescription)
}
// setup the video output to the screen and add output to our capture session
let captureOutput = AVCaptureVideoDataOutput()
captureSession.addOutput(captureOutput)
let previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
previewLayer.frame = view.frame
view.layer.addSublayer(previewLayer)
// buffer the video and start the capture session
captureOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "videoQueue"))
captureSession.startRunning()
// // creates a new capture session
// let captureSession = AVCaptureSession()
//
// // search for available capture devices
// let availableDevices = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInWideAngleCamera], mediaType: AVMediaType.video, position: .back).devices
//
// // get capture device, add device input to capture session
// do {
// if let captureDevice = availableDevices.first {
// captureSession.addInput(try AVCaptureDeviceInput(device: captureDevice))
// }
// } catch {
// print(error.localizedDescription)
// }
//
// // setup output, add output to capture session
// let captureOutput = AVCaptureVideoDataOutput()
// captureSession.addOutput(captureOutput)
//
// captureOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "videoQueue"))
//
// let previewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
// previewLayer.frame = view.frame
// previewLayer.videoGravity = .resizeAspectFill
// view.layer.addSublayer(previewLayer)
//
// captureSession.startRunning()
}
// called everytime a frame is captured
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
// load our CoreML Pokedex model
guard let model = try? VNCoreMLModel(for: model_squeezeNet_TSR().model) else { return }
// run an inference with CoreML
let request = VNCoreMLRequest(model: model) { (finishedRequest, error) in
// grab the inference results
guard let results = finishedRequest.results as? [VNClassificationObservation] else { return }
// grab the highest confidence result
guard let Observation = results.first else { return }
// create the label text components
let predclass = "\(Observation.identifier)"
let predconfidence = String(format: "%.02f%", Observation.confidence * 100)
// set the label text
DispatchQueue.main.async(execute: {
self.label.text = "\(predclass) \(predconfidence)"
})
}
// create a Core Video pixel buffer which is an image buffer that holds pixels in main memory
// Applications generating frames, compressing or decompressing video, or using Core Image
// can all make use of Core Video pixel buffers
guard let pixelBuffer: CVPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
// execute the request
try? VNImageRequestHandler(cvPixelBuffer: pixelBuffer, options: [:]).perform([request])
// guard let model = try? VNCoreMLModel(for: model_squeezeNet_TSR().model) else { return }
// let request = VNCoreMLRequest(model: model) { (finishedRequest, error) in
// guard let results = finishedRequest.results as? [VNClassificationObservation] else { return }
// guard let Observation = results.first else { return }
//
// DispatchQueue.main.async(execute: {
// self.label.text = "\(Observation.identifier)"
// print(Observation.confidence)
// })
// }
// guard let pixelBuffer: CVPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
// // executes request
// try? VNImageRequestHandler(cvPixelBuffer: pixelBuffer, options: [:]).perform([request])
}
func setupLabel() {
label.centerXAnchor.constraint(equalTo: view.centerXAnchor).isActive = true
label.bottomAnchor.constraint(equalTo: view.bottomAnchor, constant: -50).isActive = true
}
}
I don't know if this fixes it, but in your conversion script try the following:
class_labels = list(range(1, 43))
class_labels = [str(x) for x in class_labels] # add this line
Currently your class labels are integers. It's possible this confuses Core ML or Vision at some point.