How to Make the Hand Detection of Mediapipe Work on Hand with Gloves

Question

I was trying to make the hand detection of mediapipe to work on hands with blue gloves in real time. But it was not working properly. The detection can still work on gloves if the color is similar to skintone. So, I tried to do some pre-processing wherein I was changing the blue pixels found on the frame into nude color. The result was it cannot detect the hand accurately - sometimes it can but it would later disappear.

Please help me fix this. I have read somewhere that this hand detection can work on blue gloves in real time by changing the color of the glove to skin tone while retaining the shading of the hand. But I have no idea on how to do that properly. I would deeply appreciate it if you can help me T-T.

def findHands(self, img, draw=True, flipType=True):
 imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

 blue  = [0,0,255]
 nude=[225, 190, 160]

 #Make mask 
 Bmask = np.all(imgRGB == blue, axis=-1)

 #Replace blue pixel into nude
 imgRGB[Bmask] = nude

 img_res=imgRGB

 #Send frame to mediapipe        
 self.results = self.hands.process(img_res)

Answer 1

The simplest solution is to omit the cv2.cvtColor(), which will cause the image channels to swap, effectively making blue look like orange. Here is an example (for mediapipe==0.9.0):

import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands

with mp_hands.Hands(
    static_image_mode=True,
    max_num_hands=2,
    min_detection_confidence=0.5) as hands:
        image = cv2.imread("gloves.jpg")
        results = hands.process(image)

        if results.multi_hand_landmarks:
            image_height, image_width, _ = image.shape
            for hand_landmarks in results.multi_hand_landmarks:
                mp_drawing.draw_landmarks(
                    image,
                    hand_landmarks,
                    mp_hands.HAND_CONNECTIONS,
                    mp_drawing_styles.get_default_hand_landmarks_style(),
                    mp_drawing_styles.get_default_hand_connections_style())
        else:
            print("no hands were found")
                
        cv2.imshow("result", image)
        cv2.waitKey(0)

This should be sufficient for MediaPipe to recognize the hands:

Alternatively, you can also make a fine-tuning of the color change using Hue offset in HSV schema (adapted from here). This would be a "better" solution since it only changes the hue but does not swap the channels.

Edit: I've created a simple Hue-tunning "GUI-like" script that helps to find suitable Hue-offset:

import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands

window_title = 'Recognition result'
trackbar_title = 'Hue offset'

class HueHelper:
    def __init__(self):
        self.mp_hands = mp_hands.Hands(static_image_mode=True,
                                        max_num_hands=2,
                                        min_detection_confidence=0.5)
        self.img_bgr = None

    def apply_hue_offset(self,image, hue_offset):# 0 is no change; 0<=huechange<=180
        # convert img to hsv
        img_hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
        h = img_hsv[:,:,0]
        s = img_hsv[:,:,1]
        v = img_hsv[:,:,2]
        # shift the hue
        # cv2 will clip automatically to avoid color wrap-around
        img_hsv = cv2.add(h, hue_offset)
        # combine new hue with s and v
        img_hsv = cv2.merge([img_hsv,s,v])
        # convert from HSV to BGR
        return cv2.cvtColor(img_hsv, cv2.COLOR_HSV2BGR)

    def on_trackbar_change(self, trackbar_hue_offset):
        img_bgr_modified = self.recognize(self.img_bgr.copy(), trackbar_hue_offset)
        cv2.imshow(window_title, img_bgr_modified)

    def recognize(self, img_bgr, hue_offset):
        img_bgr = self.apply_hue_offset(img_bgr, hue_offset)
        img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)
        results = self.mp_hands.process(img_rgb)

        if results.multi_hand_landmarks:
            for hand_landmarks in results.multi_hand_landmarks:
                mp_drawing.draw_landmarks(
                    img_bgr,
                    hand_landmarks,
                    mp_hands.HAND_CONNECTIONS,
                    mp_drawing_styles.get_default_hand_landmarks_style(),
                    mp_drawing_styles.get_default_hand_connections_style())
        else:
            print('no hands were found')
        return img_bgr
    
    def run(self, img_path):
        self.img_bgr = cv2.imread(img_path)
        if self.img_bgr is None: print('Image was not found!')
        self.on_trackbar_change(0)
        # Hue range is 0-179: https://docs.opencv.org/4.x/df/d9d/tutorial_py_colorspaces.html
        cv2.createTrackbar(trackbar_title, window_title, 0, 179, self.on_trackbar_change)
        cv2.waitKey(0)
        cv2.destroyAllWindows()

if __name__ == '__main__':
    h = HueHelper()
    h.run('gloves.jpg')

Again, a demo: