Python to ML.NET Custom Object Detection in image

Question

I trained a custom model that can find custom objects in the image. I used a terrific article

Many thanks to Evan EdjeElectronics.

This python code works fine:

    import cv2
    import numpy as np
    import tensorflow as tf       
    
    PATH_TO_CKPT = os.path.join(CWD_PATH, 'model.pb')
       
    # Path to label map file
    PATH_TO_LABELS = os.path.join(CWD_PATH,'training','labelmap.pbtxt')
    
    # Path to image
    PATH_TO_IMAGE = "D:\\documents\\_1.jpg"
    
    # Load the Tensorflow model into memory.
    detection_graph = tf.Graph()
    with detection_graph.as_default():
        od_graph_def = tf.GraphDef()
        with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid:
            serialized_graph = fid.read()
            od_graph_def.ParseFromString(serialized_graph)
            tf.import_graph_def(od_graph_def, name='')
    
        sess = tf.Session(graph=detection_graph)
    
# Input tensor is the image
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')

# Output tensors are the detection boxes, scores, and classes
# Each box represents a part of the image where a particular object was detected
detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0')

# Each score represents level of confidence for each of the objects.
# The score is shown on the result image, together with the class label.
detection_scores = detection_graph.get_tensor_by_name('detection_scores:0')
detection_classes = detection_graph.get_tensor_by_name('detection_classes:0')

# Number of objects detected
num_detections = detection_graph.get_tensor_by_name('num_detections:0')

# Load image using OpenCV and
# expand image dimensions to have shape: [1, None, None, 3]
# i.e. a , whersingle-column arraye each item in the column has the pixel RGB value
image = cv2.imread(PATH_TO_IMAGE)
image_expanded = np.expand_dims(image, axis=0)

# Perform the actual detection by running the model with the image as input
(boxes, scores, classes, num) = sess.run(
    [detection_boxes, detection_scores, detection_classes, num_detections],
    feed_dict={image_tensor: image_expanded})

Now i am trying to use my tensorflow model in ML.NET

// For checking tensor names, you can open the TF model .pb file with tools like Netron: https://github.com/lutzroeder/netron
public struct TensorFlowModelSettings
{
    // Input tensor name.
    public const string inputTensorName = "image_tensor:0";
    // Output tensor name.
    public const string outputTensorName = "detection_boxes:0";
}

/// <summary>
/// Setup ML.NET model  by tensorFlow .pb model file
/// </summary>
/// <param name="tensorFlowModelFilePath">Full path for .pb model file</param>
private ITransformer SetupMlnetModel(string tensorFlowModelFilePath)
{
    var pipeline = _mlContext.Transforms
        .ResizeImages(
            outputColumnName: TensorFlowModelSettings.inputTensorName, 
            imageWidth: ImageSettings.imageWidth, 
            imageHeight: ImageSettings.imageHeight, 
            inputColumnName: nameof(ImageInputData.Image))
        .Append(_mlContext.Transforms.ExtractPixels(
            outputColumnName: TensorFlowModelSettings.inputTensorName, 
            interleavePixelColors: ImageSettings.channelsLast, 
            offsetImage: ImageSettings.mean))
        .Append(_mlContext.Model.LoadTensorFlowModel(tensorFlowModelFilePath)
            .ScoreTensorFlowModel(outputColumnNames: new[]                                      { TensorFlowModelSettings.outputTensorName },
                                 inputColumnNames: new[] { TensorFlowModelSettings.inputTensorName }, 
                                 addBatchDimensionInput: false));

    ITransformer mlModel = pipeline.Fit(CreateEmptyDataView());

    return mlModel;
}

Using the instruction step by step - I get an error when I call method pipeline.Fit:

System.ArgumentOutOfRangeException: 'Schema mismatch for input column 'image_tensor:0': expected Byte, got Single Arg_ParamName_Name

Link to image with code

Answer 1

I solved the problem with TensorFlowSharp

using (var graph = new TFGraph ()) {
    var model = File.ReadAllBytes (modelFile);
    graph.Import (new TFBuffer (model));

    using (var session = new TFSession (graph)) {
        Console.WriteLine("Detecting objects");

        foreach (var tuple in fileTuples) {
            //var tensor = ImageUtil.CreateTensorFromImageFile (tuple.input, TFDataType.UInt8);
            var tensor = ImageUtil.CreateTensorFromImageFileAlt (tuple.input, TFDataType.UInt8);
            var runner = session.GetRunner ();


            runner
                .AddInput (graph ["image_tensor"] [0], tensor)
                .Fetch (
                graph ["detection_boxes"] [0],
                graph ["detection_scores"] [0],
                graph ["detection_classes"] [0],
                graph ["num_detections"] [0]);
            var output = runner.Run ();

            var boxes = (float [,,])output [0].GetValue (jagged: false);
            var scores = (float [,])output [1].GetValue (jagged: false);
            var classes = (float [,])output [2].GetValue (jagged: false);
            var num = (float [])output [3].GetValue (jagged: false);

            DrawBoxes (boxes, scores, classes, tuple.input, tuple.output, MIN_SCORE_FOR_OBJECT_HIGHLIGHTING);
            Console.WriteLine($"Done. See {_output_relative}");
        }
    }
}