how to show multiple videoCapture in one frame?

Question

i have an SLR (sign language Recognition) task, and i want to show the preprocessing part, here is my code :

import numpy as np
import cv2
import keras
from keras.preprocessing.image import ImageDataGenerator
from keras.models import load_model
import tensorflow as tf

from tensorflow.compat.v1 import ConfigProto
from tensorflow.compat.v1 import InteractiveSession

config = ConfigProto()
config.gpu_options.allow_growth = True
session = InteractiveSession(config=config)


#Load CNN Model
model = load_model("VGG16withALLTRAINABLE(NO BACKGROUND).h5")

#Creating ROI frame for capturing hand
top_ROI = 100
btm_ROI = 300
right_ROI = 50
left_ROI = 250

#Creating Background Removal Parameters
blur_size = 5
canny_low = 25
# min_area = 0
# max_area = 0
canny_high = 150
dilate_iter = 10
erode_iter = 10
mask_color = (0.0,0.0,0.0)

#Video Capture
cap = cv2.VideoCapture(0)


while True:
    ret,frame = cap.read()

    #flipping frame
    # frame = cv2.flip(frame, 1)

    #Create ROI inside Frame
    roi = frame[top_ROI:btm_ROI, right_ROI:left_ROI]
    cv2.rectangle(frame, (left_ROI, top_ROI), (right_ROI,btm_ROI), (255,128,0), 3) #Visual Rectangle for ROI

    #Resizing and Reshaping to equalize model input size and shape
    roi = cv2.resize(roi, (300, 300))
    blurred_roi = cv2.GaussianBlur(roi, (blur_size,blur_size) , 0)
    gray_roi = cv2.cvtColor(blurred_roi, cv2.COLOR_BGR2GRAY)
    _,threshed = cv2.threshold(gray_roi, 100, 255, cv2.THRESH_BINARY_INV)

    # edge = cv2.Canny(gray_roi, canny_low, canny_high)
    # edge = cv2.dilate(edge, None)
    # edge = cv2.erode(edge, None)

    cntr = []
    cntr_area = []

    contours,_= cv2.findContours(threshed, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
    contour_info = []

    for c in contours:
        contour_info.append((c,cv2.contourArea(c), ))

    contour_info = np.array(contour_info)
    contour_info = sorted(contour_info, key=lambda x: x[1], reverse=True)
    max_contour = contour_info[0]

    mask = np.zeros(threshed.shape)
    cv2.fillConvexPoly(mask, max_contour[0], (255))

    mask = cv2.dilate(mask, None, iterations=dilate_iter)
    mask = cv2.erode(mask, None, iterations=erode_iter)
    mask = cv2.GaussianBlur(mask, (blur_size, blur_size), 0)
    mask_stack = np.dstack([mask]*3)    # Create 3-channel alpha mask

    #-- Blend masked img into MASK_COLOR background --------------------------------------
    mask_stack  = mask_stack.astype('float32') / 255.0          # Use float matrices, 
    roi         = roi.astype('float32') / 255.0                 #  for easy blending

    masked = (mask_stack * roi) + ((1-mask_stack) * mask_color) # Blend
    masked = (masked * 255).astype('uint8')                     # Convert back to 8-bit 

    print(mask.shape)
    print(mask_stack.shape)
    print(masked.shape)
    
    cv2.imshow("Frame", frame)
    cv2.imshow("ROI", gray_roi)
    cv2.imshow("Thresed", threshed)

    cv2.imshow('Mask', masked)



    key = cv2.waitKey(1)
    if key == 27:
        break

cap.release()
cv2.destroyAllWindows()

This is my current result [Result in diffrent Frames]

My question is, can i make all the result in one frames (one frame with multiple videos) ?

i have tried once with this code, but it wont work while i add the second video stream functions (video_stream2()) :

from tkinter import *
from PIL import ImageTk, Image
import cv2

#Creating ROI frame for capturing hand

top_ROI = 100
btm_ROI = 300
right_ROI = 50
left_ROI = 250


root = Tk()
root.geometry("1920x1080")

# Create a frame
Main_video = Frame(root, highlightbackground='grey', highlightthicknes=3)
Main_video.grid(row=0, column= 0, padx=450, pady=150, ipadx= 0, ipady=0)

Roi_video = Frame(root, highlightbackground='grey', highlightthicknes=3)
Roi_video.grid(row=0, column= 0, padx=0, pady=0, ipadx= 0, ipady=0)

# Create a label in the frame
label_main = Label(Main_video)
label_main.grid()

label_roi = Label(Roi_video)
label_roi.grid()

# Capture from camera
cap = cv2.VideoCapture(0)

# function for video streaming
def video_stream():
    _, frame = cap.read()

    #Create ROI inside Frame

    cv2image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

    #Main Video
    img = Image.fromarray(cv2image)
    imgtk = ImageTk.PhotoImage(image=img)
    label_main.imgtk = imgtk
    label_main.configure(image=imgtk)
    label_main.after(1, video_stream) 

def video_stream2():
    _, frame = cap.read()

    #Create ROI inside Frame
    roi = frame[top_ROI:btm_ROI, right_ROI:left_ROI]
    cv2.rectangle(frame, (left_ROI, top_ROI), (right_ROI,btm_ROI), (255,128,0), 3) #Visual Rectangle for ROI

    cv2roi_gray = cv2.cvtColor(roi, cv2.COLOR_RGB2GRAY)

    #Roi Video
    roi_img = Image.fromarray(cv2roi_gray)
    imgtk_roi= ImageTk.PhotoImage(image=roi_img)
    label_roi.imgtk_roi = imgtk_roi
    label_roi.configure(image=imgtk_roi)
    label_roi.after(1, video_stream2)


video_stream()
video_stream2()
root.mainloop()

Answer 1

The procedure to combine several images (windows) to one like that:

...is easy by following the example code:

import numpy as np
import cv2
import time

#Video Capture
cap = cv2.VideoCapture(0)

while(True):
    ret,frame = cap.read()

    frame_uus=cv2.resize(frame,(240,160))
    #let's simulate the images...
    #frame=np.random.randint(0,255,[320,480,3],dtype='uint8')
    gray_roi=0.5*np.random.randint(0,255,[160,240,1],dtype='uint8')+0.5*frame_uus[:,:,0:1]
    threshed=0.1+0*np.random.randint(0,255,[160,240,3],dtype='uint8')+0.3*frame_uus
    masked=0.5*np.random.randint(0,255,[160,240,3],dtype='uint8')+0.2*frame_uus

    #make sure all data is in uint8-format suitable for cv2..
    gray_roi=gray_roi.astype(np.uint8)
    threshed=threshed.astype(np.uint8)
    masked=masked.astype(np.uint8)

    #show separate images...

    cv2.imshow("Frame", frame)
    cv2.imshow("ROI", gray_roi)
    cv2.imshow("Thresed", threshed)
    cv2.imshow('Mask', masked)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

    #Define space between images...
    vali=2

    #let's combine the images...
    max_height=np.max([len(frame),len(gray_roi),len(threshed),len(masked)])
    #Let's calculate total width for the combined image...remember to add space between images...
    total_width=len(frame[0])+len(gray_roi[0])+len(threshed[0])+len(masked[0])+4*vali
    #For clearness let's make a green background image
    baseimage=np.zeros([max_height,total_width,3],'uint8')
    baseimage[:,:,1]=255

    #let's add separate images to the baseimage
    baseimage[0:len(frame),0:len(frame[0]),:]=frame

    #Take into account the grayscale...
    alku=len(frame[0])+vali
    loppu=alku+len(gray_roi[0])
    baseimage[0:len(gray_roi),alku:loppu,0:1]=gray_roi
    baseimage[0:len(gray_roi),alku:loppu,1:2]=gray_roi
    baseimage[0:len(gray_roi),alku:loppu,2:3]=gray_roi

    #Add next image...
    alku=loppu+vali
    loppu=alku+len(threshed[0])
    baseimage[0:len(threshed),alku:loppu,:]=threshed

    #And the last one...
    alku=loppu+vali
    loppu=alku+len(masked[0])
    baseimage[0:len(masked),alku:loppu,:]=masked

    #And finally let's show the baseimage...
    cv2.imshow('Combined', baseimage)


cap.release()
cv2.destroyAllWindows()