Multi-scale Template Matching in real-time

Question

I was able to do template matching from pictures from this and I applied the same to real-time which means I looped through the frames. But it does not seem to be matching the template with frames, And I realized that found(bookkeeping variable) is always None.

import cv2 as cv2
import numpy as np
import imutils


def main():

    template = cv2.imread("C:\\Users\\Manthika\\Desktop\\opencvtest\\template.jpg")
    template = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY)
    template = cv2.Canny(template, 50, 200)
    (tH, tW) = template.shape[:2]
    cv2.imshow("Template", template)

    windowName = "Something"
    cv2.namedWindow(windowName)
    cap = cv2.VideoCapture(0)

    if cap.isOpened():
        ret, frame = cap.read()
    else:
        ret = False

    # loop over the frames to find the template
    while ret:
        # load the image, convert it to grayscale, and initialize the
        # bookkeeping variable to keep track of the matched region
        ret, frame = cap.read()
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        found = None

        # loop over the scales of the image
        for scale in np.linspace(0.2, 1.0, 20)[::-1]:
            # resize the image according to the scale, and keep track
            # of the ratio of the resizing
            resized = imutils.resize(gray, width=int(gray.shape[1] * scale))
            r = gray.shape[1] / float(resized.shape[1])

            # if the resized image is smaller than the template, then break
            # from the loop
            if resized.shape[0] < tH or resized.shape[1] < tW:
                break

            # detect edges in the resized, grayscale image and apply template
            # matching to find the template in the image
            edged = cv2.Canny(resized, 50, 200)
            result = cv2.matchTemplate(edged, template, cv2.TM_CCOEFF)
            (_, maxVal, _, maxLoc) = cv2.minMaxLoc(result)

            # if we have found a new maximum correlation value, then update
            # the bookkeeping variable
            if found is None or maxVal > found[0]:
                found = (maxVal, maxLoc, r)
                print(found)

            # unpack the bookkeeping variable and compute the (x, y) coordinates
            # of the bounding box based on the resized ratio
        print(found) # here is the PROBLEM, found is always none.
        if found is None:
            # just show only the frames if the template is not detected
            cv2.imshow(windowName, frame)
        else:
            (_, maxLoc, r) = found
            (startX, startY) = (int(maxLoc[0] * r), int(maxLoc[1] * r))
            (endX, endY) = (int((maxLoc[0] + tW) * r), int((maxLoc[1] + tH) * r))

            # draw a bounding box around the detected result and display the image
            cv2.rectangle(frame, (startX, startY), (endX, endY), (0, 0, 255), 2)
            cv2.imshow(windowName, frame)

        if cv2.waitKey(1) == 27:
            break

    cv2.destroyAllWindows()
    cap.release()


if __name__ == "__main__":
    main()

Please help me to solve this problem.

The template is :

Answer 1

This is probably your problem:

if resized.shape[0] < tH or resized.shape[1] < tW:
                break

The template you use is 743x887 pixels, taller than it is wide. That shape is unlike what most camera's output, which is wider than tall. Therefor, if you're not using a 1080p fullHD camera, the output of the camera you're using is probably already smaller than the template (in height). This breaks the loop and causes found to be None. You should check this with a print statement before the break.

The solution is to check the resolution of your camera and resize your template to a smaller image accordingly.

Update:
Extra question in comments. Loop through all templates, perform matchTemplate and compare the correlation values to get the best matching template.

# create an array with template images and set variables
templates = [templ1,templ2,templ3,templ4,templ5]
curr_max = 0
index = 0

# find the best match
for i in range(len(templates)):
    # perform matchtemplate
    res= cv2.matchTemplate(img,templates[i],cv2.TM_CCOEFF)
    # get the highest correlation value of the result
    maxVal = res.max()
    # if the correlation is highest thus far, store the value and index of template
    if maxVal > curr_max:
        curr_max = maxVal
        index = i

# print index of template with highest correlation
print(index)