How can I locate and measure this object in the picture?

Question

I have this image Interest in detecting the long structure from the first image and the expected results are attached in the next image.

The expected results

I have tried the following procedures

1. thresholding
2. contour detection
3. alignment of the binary image to the horizontal (use the angle of rotation)
4. Advice needed Next from step 3 because my codes can detect the whole object but not the structure of interest

Here are my codes

import cv2
import numpy as np
import math
iterations = 7
img00 = cv2.imread('./camera1'+str(a)+'.jpg')
gray00 = cv2.cvtColor(img00, cv2.COLOR_BGR2GRAY)

median_blur1 = cv2.medianBlur(gray00, iterations) # smoothing with median blur
ret1, thresh1 = cv2.threshold(median_blur1,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU) # binarization with Otsu
# External Contour detection 
contours1, hierarchy1 = cv2.findContours(thresh1.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
try:
    cnt1 = contours1[0]
except IndexError: # Suppressing the index error to keep the camera live
    pass
# Getting the biggest contour
if len(contours1) != 0:
    # find the biggest countour (c) by the area
    c1 = max(contours1, key = cv2.contourArea)

height, width = img00.shape[:2]
center = (width/2, height/2)
x1,y1,w1,h1 = cv2.boundingRect(thresh1)
rect1 = cv2.minAreaRect(c1)
print(rect1[2])
rotate_matrix = cv2.getRotationMatrix2D(center=center, angle=rect1[2], scale=1)
binary_rotated = cv2.warpAffine(src=thresh1, M=rotate_matrix, dsize=(width, height))

contours2, hierarchy2 = cv2.findContours(binary_rotated.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnt2 = contours2[0]


x2,y2,w2,h2 = cv2.boundingRect(binary_rotated)
rect2 = cv2.minAreaRect(cnt2)
box2 = cv2.boxPoints(rect2)
box2

cv2.drawContours(binary_rotated, [box2], 0, (0, 255, 0), 2)

cv2.drawContours(img00, [box2], 0, (0, 255, 0), 2)
cv2.imshow('image', img00)
cv2.imshow('rotated_image', binary_rotated)
cv2.waitKey(0)

Answer 1

Approach:

• minAreaRect from convex hull
• some geometry to establish a local coordinate system
• taking 1-D samples along the object (warpAffine)
• finding edges to measure thickness of bars

To get the midpoint of the center bar exactly, you'd want to go along the object halfway, then scan crosswise for the center bar.

There may be some off-by-one errors in all of this, where I look for indices. You get the idea though.

I haven't bothered implementing a local maxima function, although that would make the results more accurate. I just pick the index of the first gradient element that's above a threshold. The local maximum probably encompasses multiple elements.

One of my favorite Non-Maximum Suppression approaches, for 2D, is to use dilation (morphology operation) to suppress local non-maxima. cv.dilate would be an option, or scipy.ndimage.morphology.grey_dilation. That ought to be followed up by connected components labeling because two adjacent elements may both be local maxima, when they're equal, and you wouldn't want both of them to count, but their centroid.

# imports
import numpy as np
import cv2 as cv
import matplotlib.pyplot as plt

# utility defs for later

norm = np.linalg.norm

def normalize(vec):
    return vec / norm(vec)

# read picture, binarize
im = cv.imread('aXrEv.jpg', cv.IMREAD_GRAYSCALE)
im = im[:, 100:-150] # debris in the top right corner

(level, mask) = cv.threshold(im, 128, 255, cv.THRESH_BINARY_INV | cv.THRESH_OTSU)
# imshow(mask)

# get hull
contours, hierarchy = cv.findContours(mask, cv.RETR_TREE, cv.CHAIN_APPROX_SIMPLE)
hull = cv.convexHull(np.concatenate(contours))
print(len(hull), "points in convex hull")

canvas = cv.cvtColor(mask, cv.COLOR_GRAY2BGR)
cv.drawContours(canvas, [hull], 0, (0, 0, 255), 2)
imshow(canvas)

# get box
# oriented bounding box
rrect = cv.minAreaRect(hull)
print(rrect) # (cx,cy), (w,h), angle

box = cv.boxPoints(rrect)

# roll corners so 0-1 is long edge
if norm(box[1] - box[0]) < norm(box[2] - box[1]):
    box = np.roll(box, 1, axis=0)

print(box)

canvas = cv.cvtColor(mask, cv.COLOR_GRAY2BGR)
cv.drawContours(canvas, [box.astype(int)], 0, (0, 0, 255), 2)
for i,pt in enumerate(box):
    cv.putText(canvas,
        f"[{i}]", (pt + [-10, 10]).astype(int),
        cv.FONT_HERSHEY_PLAIN, 1.8, (255, 186, 0), thickness=2)
imshow(canvas)

# local coordinate system

# assuming long edge is corner 0-1
orig = box[0].copy()
box_length = box[1] - orig
box_width = box[3] - orig
print("orig", orig)
print("vx", box_length)
print("vy", box_width)

vx = normalize(box_length)
vy = normalize(box_width)

# extend the box a little
orig -= 1*vx
box_length += 2*vx

canvas = cv.cvtColor(mask, cv.COLOR_GRAY2BGR)
cv.arrowedLine(canvas, orig.astype(int), (orig + box_length).astype(int), (0, 0, 255), 2)
cv.arrowedLine(canvas, orig.astype(int), (orig + box_width).astype(int), (0, 255, 0), 2)
imshow(canvas)

# more coordinates
# sample lines along long edge, one center and two offset to each side
orig_25 = orig + box_width * 0.25
orig_50 = orig + box_width * 0.50
orig_75 = orig + box_width * 0.75

def sample_line(im, p1, p2, ustep=1, vwidth=1):
    "take sample along line between points, 1D or 2D"
    # vstep: exercise for the reader
    origin = np.array(p1)
    vec = p2 - p1
    ulength = int(round(np.linalg.norm(vec) / ustep)) + 1
    uvec = normalize(vec) * ustep
    vvec = np.array([-uvec[1], uvec[0]])

    if vwidth > 1:
        origin -= vvec * (vwidth - 1) / 2

    # build affine transform, backwards
    M = np.eye(3)
    M[0:2, 2] = origin
    M[0:2, 0] = uvec
    M[0:2, 1] = vvec
    assert np.allclose(M[2], [0, 0, 1])

    samples = cv.warpAffine(im, M[:2], (ulength, vwidth), flags=cv.INTER_LINEAR | cv.WARP_INVERSE_MAP)
    if vwidth == 1:
        samples = samples.squeeze(axis=0)
    return samples

# taking 1-D samples
samples_25 = sample_line(mask, orig_25, orig_25 + box_length)
samples_50 = sample_line(mask, orig_50, orig_50 + box_length)
samples_75 = sample_line(mask, orig_75, orig_75 + box_length)

fig, axes = plt.subplots(3, 1, figsize=(15, 15))
axes[0].plot(samples_25)
axes[1].plot(samples_50)
axes[2].plot(samples_75)
plt.show()

grad_25 = np.gradient(samples_25)
grad_50 = np.gradient(samples_50)
grad_75 = np.gradient(samples_75)

# find edges...

# center line: first rising edge, last falling edge
center_i0 = (grad_50 >= 50).argmax()
center_i1 = len(samples_50) - (grad_50[::-1] <= -50).argmax()
print("total length:", center_i1 - center_i0)

# lower bar: first rising, first falling edge
lower_left_i0 = (grad_25 >= +50).argmax()
lower_left_i1 = (grad_25 <= -50).argmax()
lower_right_i0 = (grad_75 >= +50).argmax()
lower_right_i1 = (grad_75 <= -50).argmax()
lower_bar_width = ((lower_right_i1 - lower_right_i0) + (lower_left_i1 - lower_left_i0)) / 2
print("lower bar:",
    "left", lower_left_i1 - lower_left_i0,
    "right", lower_right_i1 - lower_right_i0,
    "avg", lower_bar_width)


# upper bar: last falling, last rising
upper_left_i0 = (grad_25[::-1] <= -50).argmax() # last falling edge
upper_left_i1 = (grad_25[::-1] >= +50).argmax() # last rising edge
upper_right_i0 = (grad_75[::-1] <= -50).argmax()
upper_right_i1 = (grad_75[::-1] >= +50).argmax()
upper_bar_width = ((upper_right_i1 - upper_right_i0) + (upper_left_i1 - upper_left_i0)) / 2
print("upper bar:",
    "left", upper_left_i1 - upper_left_i0,
    "right", upper_right_i1 - upper_right_i0,
    "avg", upper_bar_width)

print("center segment:", (center_i1 - center_i0) - lower_bar_width - upper_bar_width)

total length: 664
lower bar: left 26 right 29 avg 27.5
upper bar: left 24 right 28 avg 26.0
center segment: 610.5

def line_with_endbars(img, pt1, pt2, barLength=20, *args, **kwargs):
    "draw line with end bars"
    pt1 = np.asarray(pt1)
    pt2 = np.asarray(pt2)

    cv.line(img, pt1.astype(int), pt2.astype(int), *args, **kwargs)

    (vx, vy) = normalize(pt2 - pt1)
    n = np.array([-vy, vx])

    cv.line(img, (pt1 - n/2 * barLength).astype(int), (pt1 + n/2 * barLength).astype(int), *args, **kwargs)
    cv.line(img, (pt2 - n/2 * barLength).astype(int), (pt2 + n/2 * barLength).astype(int), *args, **kwargs)

# draw that info
canvas = cv.cvtColor(mask, cv.COLOR_GRAY2BGR)

color = (255, 186, 0)

# lower bar, left/right
cv.line(canvas,
    (orig_25 + lower_left_i0 * vx).astype(int),
    (orig_25 + lower_left_i1 * vx).astype(int),
    color=color, thickness=2)

cv.line(canvas,
    (orig_75 + lower_right_i0 * vx).astype(int),
    (orig_75 + lower_right_i1 * vx).astype(int),
    color=color, thickness=2)

# upper bar, left/right
cv.line(canvas,
    (orig_25 + box_length - upper_left_i0 * vx).astype(int),
    (orig_25 + box_length - upper_left_i1 * vx).astype(int),
    color=color, thickness=2)

cv.line(canvas,
    (orig_75 + box_length - upper_right_i0 * vx).astype(int),
    (orig_75 + box_length - upper_right_i1 * vx).astype(int),
    color=color, thickness=2)

# center bar minus end bars
subject_e0 = orig_50 + vx * (center_i0 + lower_bar_width)
subject_e1 = orig_50 + vx * (center_i1 - upper_bar_width)

print("subject length:", np.linalg.norm(subject_e1 - subject_e0))

# draw subject
# cv.line(canvas,
#     subject_e0.astype(int),
#     subject_e1.astype(int),
#     (0, 186, 0), 2)

line_with_endbars(canvas, subject_e0, subject_e1, color=(0, 0, 255), thickness=2)

#imshow(canvas)