How to detect rectangles and rotate them in the correct direction in image

Question

I have block code like this.

def crop_image(self,img):
        cnts,gray_img = self.pre_processing_img(img)
        ans_blocks = []
        x_old, y_old, w_old, h_old = 0, 0, 0, 0

        if len(cnts) > 0:
            cnts = sorted(cnts, key=self.get_x_ver1)
            for i, c in enumerate(cnts):
                x_curr, y_curr, w_curr, h_curr = cv2.boundingRect(c)
                if w_curr * h_curr > 100000 and w_curr < h_curr:
                    check_xy_min = x_curr * y_curr - x_old * y_old
                    check_xy_max = (x_curr + w_curr) * (y_curr + h_curr) - (x_old + w_old) * (y_old + h_old)

                    if len(ans_blocks) == 0:
                        cv2.imshow("test", gray_img[y_curr:y_curr + h_curr, x_curr:x_curr + w_curr])
                        cv2.waitKey(0)
                        cv2.destroyAllWindows()
                        ans_blocks.append(
                            (gray_img[y_curr:y_curr + h_curr, x_curr:x_curr + w_curr],[x_curr,y_curr,w_curr,h_curr]))
                        x_old,y_old,w_old,h_old = x_curr,y_curr,w_curr,h_curr

                    elif check_xy_min > 20000 and check_xy_max > 20000:
                        ans_blocks.append(
                            (gray_img[y_curr:y_curr + h_curr, x_curr:x_curr + w_curr],[x_curr,y_curr,w_curr,h_curr]))
                        x_old,y_old,w_old,h_old = x_curr,y_curr,w_curr,h_curr

            sorted_ans_blocks = sorted(ans_blocks, key=self.get_x)
            return sorted_ans_blocks

And the correct image test should be like this

But in some cases it returns this image depending on the input image

Sorry for the lack of information. This is the scanned photo (happiest case)

And this is the input photo (the image from camera taked) How can I detect misaligned images and rotate them to the correct position as I want them to be?

Answer 1

Here is the basic concept to warp the distorted column to match the vertical column in Python/OpenCV.

Given one column, do the following:

• Read the reference and distorted images
• Since you posted screen captures, crop the tops off and also any black on the sides
• Threshold on the black lines using cv2.inRange() on both cropped images
• Get the largest contour of the reference image
• Get its bounding box and computed the four corners in clockwise fashion from the top-left. So TL, TR, BR, BL
• Get the largest contour of the distorted image
• Compute is rotated bounding rectangle using cv2.minAreaRect()
• Then get the box point from the rotated rectangle
• Then compute the vectors from the center of the rotated rectangle to these box points and get the angle of the vector.
• Then sort the box points by their vector angle
• Then separate the sorted points into TL, TR, BR, BL order
• Then compute the perspective transform matrix from the two sets of 4 corners.
• Then warp the distorted image using the matrix to align the distorted image with the reference image.
• Then save the results

Reference Image (test1.png):

Distorted Image (test2.png):

import cv2
import numpy as np
import math

# read the reference image
ref = cv2.imread('test1.png')
rh, rw = ref.shape[:2]

# read the distorted image
dtd = cv2.imread('test2.png')
dh, dw = dtd.shape[:2]

# crop the tops which should not really be there and black sides
cref = ref[56:rh-7, 5:rw-7]
crh, crw = cref.shape[:2]
print(cref.shape)
cdtd = dtd[57:dh-5, 0:dw]
cdh, cdw = cdtd.shape[:2]
print(cdtd.shape)
print('')

# threshold on black lines
lower = (0,0,0)
upper = (175,175,175)
tref = cv2.inRange(cref, lower, upper)
lower = (0,0,0)
upper = (150,150,150)
tdtd = cv2.inRange(cdtd, lower, upper)

# get largest contour of tref and its axis aligned bounding box
contours = cv2.findContours(tref, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contours = contours[0] if len(contours) == 2 else contours[1]
ref_cntr = max(contours, key=cv2.contourArea)
xr,yr,wr,hr = cv2.boundingRect(ref_cntr)

# get 4 corners clockwise from top-left: TL, TR, BR, BL
rtl = (xr,yr)
rtr = (xr+wr,yr)
rbr = (xr+wr,yr+hr)
rbl = (xr,yr+hr)
print(rtl, rtr, rbr, rbl)
print('')
ref_cimg = cref.copy()
cv2.drawContours(ref_cimg, [ref_cntr], 0, (0,0,255), 2)

# get largest contour of tdtd and its rotated bounding box
contours = cv2.findContours(tdtd, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contours = contours[0] if len(contours) == 2 else contours[1]
dtd_cntr = max(contours, key=cv2.contourArea)
rotrect = cv2.minAreaRect(dtd_cntr)
(center), (width,height), angle = rotrect
box = cv2.boxPoints(rotrect)
boxpts = np.intp(box)
dtd_cimg = cdtd.copy()
cv2.drawContours(dtd_cimg, [dtd_cntr], 0, (0,0,255), 2)
cv2.drawContours(dtd_cimg,[boxpts],0,(0,255,0),2)

# Get box points relative to the center of rotate rect and the angles of the vector from center to point
(cx,cy) = (center)
sort_info = []
for pt in box:
    [px,py] = pt
    pxc = px - cx
    pyc = py - cy
    ang = (180/math.pi)*math.atan2(pyc,pxc)
    sort_info.append([px,py,ang])
    #print(px, py, ang)
    #print('')

# sort by angle
def takeThird(elem):
    return elem[2]
sort_info.sort(key=takeThird, reverse=False)
#print(sort_info)

# get 4 corners clockwise from top-left: TL, TR, BR, BL
x_info = []
y_info = []
for i in range(0,4):    
    [x,y,ang] = sort_info[i]
    x_info.append(x)
    y_info.append(y)
dtl = (x_info[0],y_info[0])
dtr = (x_info[1],y_info[1])
dbr = (x_info[2],y_info[2])
dbl = (x_info[3],y_info[3])
print(dtl, dtr, dbr, dbl)

# get perspective warp
ref_pts = np.float32([ [rtl], [rtr], [rbr], [rbl] ])
dtd_pts = np.float32([ [dtl], [dtr], [dbr], [dbl] ])
matrix = cv2.getPerspectiveTransform(dtd_pts, ref_pts)

# do perspective transformation setting area outside input to black
dtd_warped = cv2.warpPerspective(cdtd, matrix, (crw,crh), cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=(255,255,255))

# Save results
cv2.imwrite('test1_thresh.jpg', tref)
cv2.imwrite('test2_thresh.jpg', tdtd)
cv2.imwrite('test1_contour.jpg', ref_cimg)
cv2.imwrite('test2_contour.jpg', dtd_cimg)
cv2.imwrite('test1_cropped.jpg', cref)
cv2.imwrite('test2_warped.jpg', dtd_warped)

# show results
cv2.imshow('ref thresh', tref)
cv2.imshow('dtd thresh', tdtd)
cv2.imshow('ref contour', ref_cimg)
cv2.imshow('dtd contour', dtd_cimg)
cv2.imshow('dtd warped', dtd_warped)
cv2.waitKey(0)

Thresholded Reference Image(after crop):

Thresholded Distorted Image (after crop):

Contour Image for Reference Image:

Contour Image and Rotated Rectangle for Distorted Image:

Cropped Reference Image for comparison:

Warped Distorted image to match cropped reference:

(With your large multi-column tests with red lines, get all the contours. Then filter them by area and or perimeter to separate out the 4 main columns. Then loop over each column and do the above processing.)