OpenCV Python Error: error: (-215) (mtype == CV_8U || mtype == CV_8S) && _mask.sameSize(*psrc1) in function cv::binary_op

Question

I'm trying to overlay an image of shirt on live video stream of myself using OpenCV Python. I'm stuck at this particular error since three days:

error: (-215) (mtype == CV_8U || mtype == CV_8S) && _mask.sameSize(*psrc1) in function cv::binary_op

This errors occurs at this line:

roi_bg = cv2.bitwise_and(roi,roi,mask = mask_inv)

My code:

import cv2                              # Library for image processing
import numpy as np
imgshirt = cv2.imread('C:/Users/sayyed javed ahmed/Desktop/Humaira/Images For Programs/aureknayashirt.png',1) #original img in bgr
musgray = cv2.cvtColor(imgshirt,cv2.COLOR_BGR2GRAY) #grayscale conversion
ret, orig_mask = cv2.threshold(musgray,150 , 255, cv2.THRESH_BINARY)
orig_mask_inv = cv2.bitwise_not(orig_mask)
origshirtHeight, origshirtWidth = imgshirt.shape[:2]
face_cascade=cv2.CascadeClassifier('C:\Users\sayyed javed ahmed\Desktop\Humaira\haarcascade_frontalface_default.xml')
cap=cv2.VideoCapture(0)
while True:
    ret,img=cap.read()
    gray=cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    faces=face_cascade.detectMultiScale(gray,1.3,5)
    for (x,y,w,h) in faces:
        cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)
        shirtWidth =  3 * w  #approx wrt face width
        shirtHeight = shirtWidth * origshirtHeight / origshirtWidth #preserving aspect ratio of original image..

        # Center the shirt..just random calculations..
        x1 = x-w
        x2 =x1+3*w
        y1 = y+h
        y2 = y1+h*2


        # Check for clipping(whetehr x1 is coming out to be negative or not..)

        if x1 < 0:
            x1 = 0
        if y1 < 0:
            y1 = 0
        if x2 > 4*w:
            x2 =4*w
        if y2 > 2* h:
            y2 = x2* origshirtHeight / origshirtWidth

        print x1 #debugging
        print x2
        print y1
        print y2
        print w
        print h
        # Re-calculate the width and height of the shirt image(to resize the image when it wud be pasted)
        shirtWidth = x2 - x1
        shirtHeight = y2 - y1

        # Re-size the original image and the masks to the shirt sizes

        shirt = cv2.resize(imgshirt, (shirtWidth,shirtHeight), interpolation = cv2.INTER_AREA) #resize all,the masks you made,the originla image,everything
        mask = cv2.resize(orig_mask, (shirtWidth,shirtHeight), interpolation = cv2.INTER_AREA)
        mask_inv = cv2.resize(orig_mask_inv, (shirtWidth,shirtHeight), interpolation = cv2.INTER_AREA)

        # take ROI for shirt from background equal to size of shirt image
        roi = img[y1:y2, x1:x2]

        print shirt.size #debugginh
        print mask.size
        print mask_inv.size
        print roi.size

        print shirt.shape
        print roi.shape

        print mask.shape
        print mask_inv.shape


            # roi_bg contains the original image only where the shirt is not
            # in the region that is the size of the shirt.
        roi_bg = cv2.bitwise_and(roi,roi,mask = mask_inv)

            # roi_fg contains the image of the shirt only where the shirt is
        roi_fg = cv2.bitwise_and(shirt,shirt,mask = mask)
        print roi_bg.shape #debugging
        print roi_fg.shape

            # join the roi_bg and roi_fg
        dst = cv2.add(roi_bg,roi_fg)
        print dst.shape
            # place the joined image, saved to dst back over the original image
        roi = dst


        break
    cv2.imshow('img',img)
    if cv2.waitKey(1) == ord('q'):
        break;
cap.release()                           # Destroys the cap object
cv2.destroyAllWindows()                 # Destroys all the windows created by imshow

I've read this thread: http://www.stackoverflow.com/questions/30117740/opencv-error-assertion-failed-mask-size-src1-size-in-binary-op but haven't grasped much. I know that the size of roi and shirt should be same,I printed the values to check if they're the same but they aren't. According to me the statements:

roi = img[y1:y2, x1:x2]

and

shirt = cv2.resize(imgshirt, (shirtWidth,shirtHeight), interpolation = cv2.INTER_AREA)

should make both their sizes as x2-x1 and y2-y1 but that's not happening. Been scratching my head over this one line since three days,any help is appreciated!

Answer 1

The error most likely came from where you fiddled with x and y variables and shirt size without making sure they fit inside the frame of the webcam feed.

I reworked your code into a working one:

import cv2
import numpy as np
imgshirt = cv2.imread('shirt.png',1)
musgray = cv2.cvtColor(imgshirt,cv2.COLOR_BGR2GRAY) #grayscale conversion
ret, orig_mask = cv2.threshold(musgray,150 , 255, cv2.THRESH_BINARY)
orig_mask_inv = cv2.bitwise_not(orig_mask)
origshirtHeight, origshirtWidth = imgshirt.shape[:2]
face_cascade=cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
cap=cv2.VideoCapture(0)
ret,img=cap.read()
img_h, img_w = img.shape[:2]
while True:
    ret,img=cap.read()
    gray=cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    faces=face_cascade.detectMultiScale(gray,1.3,5)
    for (x,y,w,h) in faces:
        cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)

        face_w = w
        face_h = h
        face_x1 = x
        face_x2 = face_x1 + face_h
        face_y1 = y
        face_y2 = face_y1 + face_h

        # set the shirt size in relation to tracked face
        shirtWidth = 3 * face_w
        shirtHeight = int(shirtWidth * origshirtHeight / origshirtWidth)


        shirt_x1 = face_x2 - int(face_w/2) - int(shirtWidth/2) #setting shirt centered wrt recognized face
        shirt_x2 = shirt_x1 + shirtWidth
        shirt_y1 = face_y2 + 5 # some padding between face and upper shirt. Depends on the shirt img
        shirt_y2 = shirt_y1 + shirtHeight

        # Check for clipping
        if shirt_x1 < 0:
            shirt_x1 = 0
        if shirt_y1 < 0:
            shirt_y1 = 0
        if shirt_x2 > img_w:
            shirt_x2 = img_w
        if shirt_y2 > img_h:
            shirt_y2 = img_h

        shirtWidth = shirt_x2 - shirt_x1
        shirtHeight = shirt_y2 - shirt_y1
        if shirtWidth < 0 or shirtHeight < 0:
            continue

        # Re-size the original image and the masks to the shirt sizes
        shirt = cv2.resize(imgshirt, (shirtWidth,shirtHeight), interpolation = cv2.INTER_AREA) #resize all,the masks you made,the originla image,everything
        mask = cv2.resize(orig_mask, (shirtWidth,shirtHeight), interpolation = cv2.INTER_AREA)
        mask_inv = cv2.resize(orig_mask_inv, (shirtWidth,shirtHeight), interpolation = cv2.INTER_AREA)

        # take ROI for shirt from background equal to size of shirt image
        roi = img[shirt_y1:shirt_y2, shirt_x1:shirt_x2]


        # roi_bg contains the original image only where the shirt is not
        # in the region that is the size of the shirt.
        roi_bg = cv2.bitwise_and(roi,roi,mask = mask)
        roi_fg = cv2.bitwise_and(shirt,shirt,mask = mask_inv)
        dst = cv2.add(roi_bg,roi_fg)
        img[shirt_y1:shirt_y2, shirt_x1:shirt_x2] = dst


        break
    cv2.imshow('img',img)
    if cv2.waitKey(1) == ord('q'):
        break;
cap.release() # Destroys the cap object
cv2.destroyAllWindows() # Destroys all the windows created by imshow

I renamed variables to make them easier to grasp. I set the shirt image and haarcascade XML paths to working directory for local testing. I also switched masks when creating roi bg and fg, not exactly sure why that was necessary, but that gave the correct outcome. Finally added img[shirt_y1:shirt_y2, shirt_x1:shirt_x2] = dst to actually paste the shirt into the video frame.

Another thing to note is when working with numpy images, always convert any division results into ints.

face recognition with shirt webcam capture

Answer 2

Have you made sure that the overlayed image won't go over the foreground size because that is usually what causes the mask sizes to differ