Transform and display cropped image in same orientation and dimension as reference image with OpenCV Python

Question

I have a reference image and I am finding that reference image within a larger test image using ORB detection. Once found, I want to save ONLY the area within the crop as a new image. I'd like to transform it into the same dimensions as the reference image and save it in the same orientation.

So far I have matched the ref image in the larger image and have masked it out. But I cant figure out how to display ONLY the cropped area as its own picture, in the proper orientation and dimensions. I want to save whats in the crop and get rid of the rest of it.

Any help would be greatly appreciated. Thank you.

import cv2
import numpy as np

#minimum ORB matches required to make a match
MIN_MATCH_COUNT = 10

img1 = cv2.imread("reference.jpg")
img2 = cv2.imread("1.jpg")

orb = cv2.ORB_create()
kp1, des1 = orb.detectAndCompute(img1,None)
kp2, des2 = orb.detectAndCompute(img2,None)

bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)
matches = bf.match(des1, des2, None)

# sorts matches
good = []
for i, m in enumerate(matches):
    if i < len(matches) - 1 and m.distance < 0.7 * matches[i+1].distance:
        good.append(m)

if len(good)>MIN_MATCH_COUNT:
    src_pts = np.float32([ kp1[m.queryIdx].pt for m in good ]).reshape(-1,1,2)
    dst_pts = np.float32([ kp2[m.trainIdx].pt for m in good ]).reshape(-1,1,2)
    M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC,5.0)
    matchesMask = mask.ravel().tolist()
    h,w,d = img1.shape
    pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)
    dst = cv2.perspectiveTransform(pts,M)
    mask = np.ones(img2.shape[:2], dtype="uint8") * 255
    rect = cv2.minAreaRect(dst)
    box = cv2.boxPoints(rect)
    box = np.int0(box)
    new = cv2.drawContours(mask, [box], -1, 0, -1)
    # remove the contours from the image and show the resulting images
    img = cv2.bitwise_and(img2, img2, mask=mask)
    cv2.imshow("Mask", mask)
    cv2.imshow("After", img)
else:
    print( "Not enough matches are found - {}/{}".format(len(good), MIN_MATCH_COUNT) )
    matchesMask = None

#This is for drawing the match lines inbetween the ref and 1.jpg images
draw_params = dict(matchColor = (0,255,0), # draw matches in green color
               singlePointColor = None,
               matchesMask = matchesMask, # draw only inliers
               flags = 2)
img3 = cv2.drawMatches(img1,kp1,img2,kp2,good,None,**draw_params)
cv2.imshow("Matches", img3)

Answer 1

You're on the right track and already did most of the work. Since you have found the ROI mask, you can perform a perspective transform to get the correct orientation. Afterwards, you can resize the image to match your reference/template image.

First we invert the mask to get the desired ROI in white and then find contours on this mask. From here we find the corners using cv2.arcLength() and cv2.approxPolyDP(). Next we perspective transform to get this

template shape: (210, 236, 3)
transformed shape: (288, 279, 3)

Now that we have the correct orientation, we just need to simply resize to match the template image. Here's the result (left) and template image (right)

template shape: (210, 236, 3)
resized shape: (210, 236, 3)

Code

import cv2
import numpy as np

def perspective_transform(image, corners):
    def order_corner_points(corners):
        # Separate corners into individual points
        # Index 0 - top-right
        #       1 - top-left
        #       2 - bottom-left
        #       3 - bottom-right
        corners = [(corner[0][0], corner[0][1]) for corner in corners]
        top_r, top_l, bottom_l, bottom_r = corners[0], corners[1], corners[2], corners[3]
        return (top_l, top_r, bottom_r, bottom_l)

    # Order points in clockwise order
    ordered_corners = order_corner_points(corners)
    top_l, top_r, bottom_r, bottom_l = ordered_corners

    # Determine width of new image which is the max distance between 
    # (bottom right and bottom left) or (top right and top left) x-coordinates
    width_A = np.sqrt(((bottom_r[0] - bottom_l[0]) ** 2) + ((bottom_r[1] - bottom_l[1]) ** 2))
    width_B = np.sqrt(((top_r[0] - top_l[0]) ** 2) + ((top_r[1] - top_l[1]) ** 2))
    width = max(int(width_A), int(width_B))

    # Determine height of new image which is the max distance between 
    # (top right and bottom right) or (top left and bottom left) y-coordinates
    height_A = np.sqrt(((top_r[0] - bottom_r[0]) ** 2) + ((top_r[1] - bottom_r[1]) ** 2))
    height_B = np.sqrt(((top_l[0] - bottom_l[0]) ** 2) + ((top_l[1] - bottom_l[1]) ** 2))
    height = max(int(height_A), int(height_B))

    # Construct new points to obtain top-down view of image in 
    # top_r, top_l, bottom_l, bottom_r order
    dimensions = np.array([[0, 0], [width - 1, 0], [width - 1, height - 1], 
                    [0, height - 1]], dtype = "float32")

    # Convert to Numpy format
    ordered_corners = np.array(ordered_corners, dtype="float32")

    # Find perspective transform matrix
    matrix = cv2.getPerspectiveTransform(ordered_corners, dimensions)

    # Return the transformed image
    return cv2.warpPerspective(image, matrix, (width, height))

#minimum ORB matches required to make a match
MIN_MATCH_COUNT = 10

img1 = cv2.imread("reference.jpg")
img2 = cv2.imread("1.jpg")

orb = cv2.ORB_create()
kp1, des1 = orb.detectAndCompute(img1,None)
kp2, des2 = orb.detectAndCompute(img2,None)

bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)
matches = bf.match(des1, des2, None)

# sorts matches
good = []
for i, m in enumerate(matches):
    if i < len(matches) - 1 and m.distance < 0.7 * matches[i+1].distance:
        good.append(m)

if len(good)>MIN_MATCH_COUNT:
    src_pts = np.float32([ kp1[m.queryIdx].pt for m in good ]).reshape(-1,1,2)
    dst_pts = np.float32([ kp2[m.trainIdx].pt for m in good ]).reshape(-1,1,2)
    M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC,5.0)
    matchesMask = mask.ravel().tolist()
    h,w,d = img1.shape
    pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)
    dst = cv2.perspectiveTransform(pts,M)
    mask = np.ones(img2.shape[:2], dtype="uint8") * 255
    rect = cv2.minAreaRect(dst)
    box = cv2.boxPoints(rect)
    box = np.int0(box)
    new = cv2.drawContours(mask, [box], -1, 0, -1)
    # remove the contours from the image and show the resulting images
    img = cv2.bitwise_and(img2, img2, mask=mask)
    mask = 255 - mask
    cv2.imshow("After", img)
else:
    print( "Not enough matches are found - {}/{}".format(len(good), MIN_MATCH_COUNT) )
    matchesMask = None

#This is for drawing the match lines inbetween the ref and 1.jpg images
draw_params = dict(matchColor = (0,255,0), # draw matches in green color
               singlePointColor = None,
               matchesMask = matchesMask, # draw only inliers
               flags = 2)
img3 = cv2.drawMatches(img1,kp1,img2,kp2,good,None,**draw_params)
cv2.imshow("Matches", img3)

cv2.imshow("Mask", mask)

# Find contour on mask and perform perspective transform
cnts = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]

for c in cnts:
    peri = cv2.arcLength(c, True)
    approx = cv2.approxPolyDP(c, 0.015 * peri, True)
    if len(approx) == 4:
        transformed = perspective_transform(img2, approx)

cv2.imshow("transformed", transformed)

print('template shape:', img1.shape)
print('transformed shape:',transformed.shape)

resized = cv2.resize(transformed, (img1.shape[1], img1.shape[0]))
cv2.imshow("resized", resized)
print('template shape:', img1.shape)
print('resized shape:',resized.shape)
cv2.waitKey()