How do you calculate the area metric when comparing three images using python?

Question

I have three images: A, B, and C. I want to compare the difference in area between (A, B) and (C, B). Is there a more efficient and accurate way to do this using Python?

Here's what I am currently doing, which I believe is quite inaccurate:

from PIL import Image
from pycocotools import mask as coco_mask
import pandas as pd, json, cv2, numpy as np, matplotlib.pyplot as plt, os

A = r"/path/to/A"
B = r"/path/to/B"
C = r"/path/to/C"

for a_path in os.listdir(A):
    for b_path in os.listdir(B):
        if b_path in a_path:
            a_image = Image.open(A + os.sep + a_path).convert('L')
            b_image = Image.open(B + os.sep + b_path).convert('L')
            a_array = np.array(a_image)
            b_array = np.array(b_image)
            overlap_array = np.bitwise_and(a_array, b_array)
            overlap_pixel_count = np.sum(overlap_array > 0)
            n_pixel_count = np.sum(b_array > 0)
            overlap_percentage = (overlap_pixel_count / n_pixel_count) * 100
            print(overlap_percentage)
        else: 
            pass


for c_path in os.listdir(C):
    for b_path in os.listdir(B):
        if b_path in c_path:
            c_image = Image.open(bounding_boxes_path + os.sep + c_path).convert('L')  # Convert to grayscale
            b_image = Image.open(segment_masks_path + os.sep + b_path).convert('L')  # Convert to grayscale
            c_array = np.array(c_image)
            b_array = np.array(b_image)
            overlap_array = np.bitwise_and(c_array, b_array)
            overlap_pixel_count = np.sum(overlap_array > 0)
            n_pixel_count = np.sum(b_array > 0)
            overlap_percentage = (overlap_pixel_count / n_pixel_count) * 100
            print(overlap_percentage)
        else: 
            pass

A:

B:

C:

Answer 1

I think you are actually trying to see how well the shapes in the images match each other, so you can use the Jaccard Index, a.k.a. "Intersection over Union"

These diagrams from Adrian Rosebrock should help understand it:

I would code it something like this:

#!/usr/bin/env python3

import cv2 as cv
import numpy as np

# Load images as greyscale and make spare empty channel
imA = cv.imread('a.png', cv.IMREAD_GRAYSCALE)
imB = cv.imread('b.png', cv.IMREAD_GRAYSCALE)
imC = cv.imread('c.png', cv.IMREAD_GRAYSCALE)
blk = np.zeros_like(imA)

# Make Boolean mask of each and count set pixels
maskA = imA > 127
maskB = imB > 127
maskC = imC > 127
cntA = np.count_nonzero(maskA)
cntB = np.count_nonzero(maskB)
cntC = np.count_nonzero(maskC)
print(f'{cntA=}, {cntB=}, {cntC=}')

# Do logical combinations and Jaccard Index for A and B
intsec = maskA & maskB
union  = maskA | maskB
cntIntsec = np.count_nonzero(intsec)
cntUnion  = np.count_nonzero(union)
Jaccard   = (cntIntsec*100.)/cntUnion
cv.imwrite('DEBUG-ABintsec.png', intsec*255)
cv.imwrite('DEBUG-ABunion.png', union*255)
visual = np.dstack((intsec*255, blk, union*255))
cv.imwrite('DEBUG-AB.png', visual)
print(f'{cntIntsec=}, {cntUnion=}, {Jaccard=}')

# Do logical combinations and Jaccard Index for B and C
intsec = maskB & maskC
union  = maskB | maskC
cntIntsec = np.count_nonzero(intsec)
cntUnion  = np.count_nonzero(union)
Jaccard   = (cntIntsec*100.)/cntUnion
cv.imwrite('DEBUG-BCintsec.png', intsec*255)
cv.imwrite('DEBUG-BCunion.png', union*255)
visual = np.dstack((intsec*255, blk, union*255))
cv.imwrite('DEBUG-BC.png', visual)
print(f'{cntIntsec=}, {cntUnion=}, {Jaccard=}')

That gives the following output, showing A and B are 15% similar whilst B and C are 68% similar:

cntA=32611, cntB=24792, cntC=23281
cntIntsec=7684, cntUnion=49719, Jaccard=15.45485629236308
cntIntsec=19503, cntUnion=28570, Jaccard=68.26391319565978

I have visualised the A-B relationship as follows:

• on the left, the union of A and B,
• in the centre, the intersection of A and B,
• on the right, the two overlaid where A is red, B is in blue and the intersection is magenta:

And the B-C relationship:

• on the left, the union of B and C,
• in the centre, the intersection of B and C,
• on the right, the two overlaid where B is red, C is in blue and the intersection is magenta:

Hopefully you can see more magenta and less individual red or blue means higher Jaccard Index.

---

I am not sure you realise your third (C) image is non-binary (i.e. it contains many shades of grey other than pure black and white), unlike A and B which are pure black and white. Is it adequate to threshold C at 50%?