Warping a license plate image to be frontal-parallel

Question

I am trying to take an image of a license plate so that I can then do some image processing to draw contours around the plate, which I can then use to warp the perspective to then view the plate face on. Unfortunately, I am getting an error which occurs when I am trying to draw contours around an image I have processed. Specifically, I get an Invalid shape (4, 1, 2) for the image data error. I am not too sure how I can go about solving this as I know that all the other images I have processed are fine. It's just when I try to draw contours something is going wrong.

import cv2
import numpy as np
from matplotlib import pyplot as plt

kernel = np.ones((3,3))
image = cv2.imread('NoPlate0.jpg')

def getContours(img):
    biggest = np.array([])
    maxArea = 0

    contours, hierarchy = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
    for cnt in contours:
        area = cv2.contourArea(cnt)
        if area > 500:
            cv2.drawContours(imgContour, cnt, -1, (255, 0, 0), 3)
            peri = cv2.arcLength(cnt, True)
            approx = cv2.approxPolyDP(cnt,0.02*peri, True)
            if area > maxArea and len(approx) == 4:
                biggest = approx
                maxArea = area
    return biggest

imgGray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
imgBlur = cv2.GaussianBlur(imgGray,(5,5),1)
imgCanny = cv2.Canny(imgBlur,150,200)
imgDial = cv2.dilate(imgCanny,kernel,iterations=2)
imgThres = cv2.erode(imgDial,kernel,iterations=2)
imgContour = image.copy()

titles = ['original', 'Blur', 'Canny', 'Dialte', 'Threshold', 'Contours' ]
images = [image,  imgBlur, imgCanny, imgDial, imgThres, getContours(imgThres)]

for i in range(6):
    plt.subplot(3, 3, i+1), plt.imshow(images[i], 'gray')
    plt.title(titles[i])

plt.show()

The exact error I am getting is this:

TypeError: Invalid shape (4, 1, 2) for image data

I am using the following image below as my input:

Answer 1

Your function only returns the actual points along the contour, which you then try to call plt.imshow on. This is why you are getting this error. What you need to do is use cv2.drawContour with this contour to get what you want. In this case, we should restructure your getContours function so that it returns the both the coordinates (so you can use this for later) and the actual contours drawn on the image itself. Instead of mutating imgContour and treating it like a global variable, only draw to this image once which will be the largest contour found in the loop:

def getContours(img):
    biggest = np.array([])
    maxArea = 0
    imgContour = img.copy()  # Change - make a copy of the image to return
    contours, hierarchy = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
    index = None
    for i, cnt in enumerate(contours):  # Change - also provide index
        area = cv2.contourArea(cnt)
        if area > 500:
            peri = cv2.arcLength(cnt, True)
            approx = cv2.approxPolyDP(cnt,0.02*peri, True)
            if area > maxArea and len(approx) == 4:
                biggest = approx
                maxArea = area
                index = i  # Also save index to contour

    if index is not None: # Draw the biggest contour on the image
        cv2.drawContours(imgContour, contours, index, (255, 0, 0), 3)

    return biggest, imgContour  # Change - also return drawn image

Finally we can use this in your overall code in the following way:

import cv2
import numpy as np
from matplotlib import pyplot as plt

kernel = np.ones((3,3))
image = cv2.imread('NoPlate0.jpg')

imgGray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
imgBlur = cv2.GaussianBlur(imgGray,(5,5),1)
imgCanny = cv2.Canny(imgBlur,150,200)
imgDial = cv2.dilate(imgCanny,kernel,iterations=2)
imgThres = cv2.erode(imgDial,kernel,iterations=2)
biggest, imgContour = getContours(imgThres)  # Change

titles = ['original', 'Blur', 'Canny', 'Dilate', 'Threshold', 'Contours']
images = [image,  imgBlur, imgCanny, imgDial, imgThres, imgContour]  # Change

for i in range(6):
    plt.subplot(3, 3, i+1), plt.imshow(images[i], 'gray')
    plt.title(titles[i])

plt.show()

---

As a final note, if you want to warp the license plate image so that it's parallel to the image plane, you can use cv2.getPerspectiveTransform to define a homography going from the original source image (the source points) to the warped image (the destination points), then use cv2.warpPerspective to finally warp the image. Take note that the way the source and destination points is such that they need to be ordered so that their corresponding locations match in perspective. That is, if the first point of the set of points defining the quadrilateral of your region was the top left, the source and destination points should both be defining the top left corner. You can do this by finding the centroid of the quadrilaterals for both the source and destination, then finding the angle subtended from the centroid to each of the corners and ordering both of them that way by sorting the angles.

Here's the following function I wrote that does this called order_points:

def order_points(pts):
    # Step 1: Find centre of object
    center = np.mean(pts)

    # Step 2: Move coordinate system to centre of object
    shifted = pts - center

    # Step #3: Find angles subtended from centroid to each corner point
    theta = np.arctan2(shifted[:, 0], shifted[:, 1])

    # Step #4: Return vertices ordered by theta
    ind = np.argsort(theta)
    return pts[ind]

Finally, with the corner points you returned, try doing:

src = np.squeeze(biggest).astype(np.float32) # Source points
height = image.shape[0]
width = image.shape[1]
# Destination points
dst = np.float32([[0, 0], [0, height - 1], [width - 1, 0], [width - 1, height - 1]])

# Order the points correctly
src = order_points(src)
dst = order_points(dst)

# Get the perspective transform
M = cv2.getPerspectiveTransform(src, dst)

# Warp the image
img_shape = (width, height)
warped = cv2.warpPerspective(img, M, img_shape, flags=cv2.INTER_LINEAR)

src are the four corners of the source polygon that encompasses the license plate. Take note because they're returned from cv2.approxPolyDP, they will be a 4 x 1 x 2 NumPy array of integers. You will need to remove the singleton second dimension and convert these into 32-bit floating-point so that they can be used with cv2.getPerspectiveTransform. dst are the destination points where each of the corners in the source polygon get mapped to the corner points of actual output image dimensions, which will be the same size as the input image. One last thing to remember is that with cv2.warpPerspective, you specify the size of the image as (width, height).

If you finally want to integrate this all together and make the getContours function return the warped image, we can do this very easily. We have to modify a few things to get this to work as intended:

1. getContours will also take in the original RGB image so that we can properly visualise the contour and get a better perspective on how the license plate is being localised.
2. Add in the logic to warp the image inside getContours as I showed above.
3. Change the plotting code to also include this warped image as well as return the warped image from getContours.
4. Modify the plotting code slightly for showing the original image in Matplotlib, as cv2.imread reads in images in BGR format, but Matplotlib expects images to be in RGB format.

Therefore:

import cv2
import numpy as np
from matplotlib import pyplot as plt

def order_points(pts):
    # Step 1: Find centre of object
    center = np.mean(pts)

    # Step 2: Move coordinate system to centre of object
    shifted = pts - center

    # Step #3: Find angles subtended from centroid to each corner point
    theta = np.arctan2(shifted[:, 0], shifted[:, 1])

    # Step #4: Return vertices ordered by theta
    ind = np.argsort(theta)
    return pts[ind]

def getContours(img, orig):  # Change - pass the original image too
    biggest = np.array([])
    maxArea = 0
    imgContour = orig.copy()  # Make a copy of the original image to return
    contours, hierarchy = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
    index = None
    for i, cnt in enumerate(contours):  # Change - also provide index
        area = cv2.contourArea(cnt)
        if area > 500:
            peri = cv2.arcLength(cnt, True)
            approx = cv2.approxPolyDP(cnt,0.02*peri, True)
            if area > maxArea and len(approx) == 4:
                biggest = approx
                maxArea = area
                index = i  # Also save index to contour

    warped = None  # Stores the warped license plate image
    if index is not None: # Draw the biggest contour on the image
        cv2.drawContours(imgContour, contours, index, (255, 0, 0), 3)

        src = np.squeeze(biggest).astype(np.float32) # Source points
        height = image.shape[0]
        width = image.shape[1]
        # Destination points
        dst = np.float32([[0, 0], [0, height - 1], [width - 1, 0], [width - 1, height - 1]])

        # Order the points correctly
        biggest = order_points(src)
        dst = order_points(dst)

        # Get the perspective transform
        M = cv2.getPerspectiveTransform(src, dst)

        # Warp the image
        img_shape = (width, height)
        warped = cv2.warpPerspective(orig, M, img_shape, flags=cv2.INTER_LINEAR)

    return biggest, imgContour, warped  # Change - also return drawn image

kernel = np.ones((3,3))
image = cv2.imread('NoPlate0.jpg')

imgGray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
imgBlur = cv2.GaussianBlur(imgGray,(5,5),1)
imgCanny = cv2.Canny(imgBlur,150,200)
imgDial = cv2.dilate(imgCanny,kernel,iterations=2)
imgThres = cv2.erode(imgDial,kernel,iterations=2)
biggest, imgContour, warped = getContours(imgThres, image)  # Change

titles = ['Original', 'Blur', 'Canny', 'Dilate', 'Threshold', 'Contours', 'Warped']  # Change - also show warped image
images = [image[...,::-1],  imgBlur, imgCanny, imgDial, imgThres, imgContour, warped]  # Change

# Change - Also show contour drawn image + warped image
for i in range(5):
    plt.subplot(3, 3, i+1)
    plt.imshow(images[i], cmap='gray')
    plt.title(titles[i])

plt.subplot(3, 3, 6)
plt.imshow(images[-2])
plt.title(titles[-2])

plt.subplot(3, 3, 8)
plt.imshow(images[-1])
plt.title(titles[-1])

plt.show()

The figure I get is now: