How can I perform Template Matching process in SUB-IMAGE extracted from ORIGINAL-IMAGE and Display the results in Original Image

Question

One whole day I have tried a lot to get all the related matches (with matchtemplate function) in sub-Image , which is ROI i have already extracted from the original image with the mousecallback function. So my code is below for the Matchingfunction

 ////Matching Function
void CTemplate_MatchDlg::OnBnTemplatematch()
 {

  namedWindow("reference",CV_WINDOW_AUTOSIZE);    
   while(true)
   { 

 Mat ref = imread("img.jpg");                    //  Original Image   
 mod_ref = cvCreateMat(ref.rows,ref.cols,CV_32F);// resizing the image to fit in picture box
 resize(ref,mod_ref,Size(),0.5,0.5,CV_INTER_AREA);

   Mat tpl =imread("Template.jpg"); // TEMPLATE IMAGE  

  cvSetMouseCallback("reference",find_mouseHandler,0);

  Mat aim=roiImg1.clone(); // SUB_IMAGE FROM ORIGINALIMAGE                   
                               // aim variable contains the ROI matrix
                               // next, want to perform template matching in that ROI                                                //                                     and display results on original image 


     if(select_flag1 == 1)
    {

        // imshow("ref",aim);

        Mat res(aim.rows-tpl.rows+1, aim.cols-tpl.cols+1,CV_32FC1);
                    matchTemplate(aim, tpl, res, CV_TM_CCOEFF_NORMED);
        threshold(res, res, 0.8, 1., CV_THRESH_TOZERO);

     while (1) 
   {
    double minval, maxval, threshold = 0.8;
    Point minloc, maxloc;
    minMaxLoc(res, &minval, &maxval, &minloc, &maxloc);

   //// Draw Bound boxes for detected templates in sub matrix

    if (maxval >= threshold)
     {
        rectangle(
            aim, 
            maxloc, 
            Point(maxloc.x + tpl.cols, maxloc.y + tpl.rows), 
            CV_RGB(0,255,0), 1,8,0
        );
        floodFill(res, maxloc, cv::Scalar(0), 0, cv::Scalar(.1), cv::Scalar(1.));
          }else
        break;
        }
     }
            ////Bounding box for ROI  selection with mouse

      rectangle(mod_ref, rect2, CV_RGB(255, 0, 0), 1, 8, 0);  // rect2 is ROI 
                       // my idea is to get all the matches in ROI with bounding boxes
                       // no need to mark any matches outside the ROI  
                       //Clearly i want to process only ROI  

    imshow("reference", mod_ref); // show the image with the results 
    waitKey(10);
    }
 //cvReleaseMat(&mod_ref);
 destroyWindow("reference");


}

/// ImplementMouse Call Back

void find_mouseHandler(int event, int x, int y, int flags, void* param)

{
if (event == CV_EVENT_LBUTTONDOWN && !drag)
{
    /* left button clicked. ROI selection begins*/
    point1 = Point(x, y);
    drag = 1;

}

if (event == CV_EVENT_MOUSEMOVE && drag)
{
    /* mouse dragged. ROI being selected*/ 
    Mat img3 = mod_ref.clone();
    point2 = Point(x, y);
    rectangle(img3, point1, point2, CV_RGB(255, 0, 0), 1, 8, 0);
    imshow("reference", img3);

    //  
}

if (event == CV_EVENT_LBUTTONUP && drag)
{

    Mat img4=mod_ref.clone();
            point2 = Point(x, y);
    rect1 = Rect(point1.x,point1.y,x-point1.x,y-point1.y);
            drag = 0;
    roiImg1 = mod_ref(rect1);  //SUB_IMAGE MATRIX
        imshow("reference", img4);
}

if (event == CV_EVENT_LBUTTONUP)
{
   /* ROI selected */
    select_flag1 = 1;
    drag = 0;
}
}

build and debugging process successfully done. But, when I click the Match button in dialog I'm getting the error:

Unhandled exception at 0x74bf812f in Match.exe: Microsoft C++ exception: cv::Exception at memory location 0x001ae150.. 

So my idea is to get all the matches in the Sub-image when compare with the TEMPLATE IMAGE and show the final result (matches with bounding boxes) in the ORIGINAL IMAGE itself.

Anyone help me in this regard!! Help would be appreciated greatly!!

Answer 1

My code below is a modification of the original tutorial provided by OpenCV.

It loads an image from the command-line and displays it on the screen so the user can draw a rectangle somewhere to select the sub-image to be the template. After that operation is done, the sub-image will be inside a green rectangle:

Press any key to let the program perform the template matching. A new window titled "Template Match:" appears displaying the original image plus a blue rectangle that shows the matched area:

#include <cv.h>
#include <highgui.h>
#include <iostream>


const char* ref_window = "Draw rectangle to select template";
std::vector<cv::Point> rect_points;


void mouse_callback(int event, int x, int y, int flags, void* param)
{
    if (!param)
        return;

    cv::Mat* ref_img = (cv::Mat*) param;

    // Upon LMB click, store the X,Y coordinates to define a rectangle.
    // Later this info is used to set a ROI in the reference image.
    switch (event)
    {
        case CV_EVENT_LBUTTONDOWN:
        {
            if (rect_points.size() == 0)
                rect_points.push_back(cv::Point(x, y));
        }
        break;

        case CV_EVENT_LBUTTONUP:
        {
            if (rect_points.size() == 1)
                rect_points.push_back(cv::Point(x, y));
        }
        break;

        default:
        break;
    }

    if (rect_points.size() == 2)
    {
        cv::rectangle(*ref_img, 
                      rect_points[0], 
                      rect_points[1], 
                      cv::Scalar(0, 255, 0),
                      2);

        cv::imshow(ref_window, *ref_img);
    }
}

int main(int argc, char* argv[])
{
    if (argc < 2)
    {
        std::cout << "Usage: " << argv[0] << " <image>" << std::endl;
        return -1;
    }

    cv::Mat source = cv::imread(argv[1]);   // original image
    if (source.empty())
    {
        std::cout << "!!! Failed to load source image." << std::endl;
        return -1;
    }

    // For testing purposes, our template image will be a copy of the original.
    // Later we will present it in a window to the user, and he will select a region 
    // as a template, and then we'll try to match that to the original image.

    cv::Mat reference = source.clone(); 

    cv::namedWindow(ref_window, CV_WINDOW_AUTOSIZE);
    cv::setMouseCallback(ref_window, mouse_callback, (void*)&reference);

    cv::imshow(ref_window, reference);
    cv::waitKey(0);

    if (rect_points.size() != 2)
    {
        std::cout << "!!! Oops! You forgot to draw a rectangle." << std::endl;
        return -1;
    }

    // Create a cv::Rect with the dimensions of the selected area in the image
    cv::Rect template_roi = cv::boundingRect(rect_points);

    // Create THE TEMPLATE image using the ROI from the rectangle
    cv::Mat template_img = cv::Mat(source, template_roi);

    // Create the result matrix
    int result_cols =  source.cols - template_img.cols + 1;
    int result_rows = source.rows - template_img.rows + 1;
    cv::Mat result;

    // Do the matching and normalize
    cv::matchTemplate(source, template_img, result, CV_TM_CCORR_NORMED);
    cv::normalize(result, result, 0, 1, cv::NORM_MINMAX, -1, cv::Mat());

    /// Localizing the best match with minMaxLoc
    double min_val = 0, max_val = 0; 
    cv::Point min_loc, max_loc, match_loc;
    int match_method = CV_TM_CCORR_NORMED;
    cv::minMaxLoc(result, &min_val, &max_val, &min_loc, &max_loc, cv::Mat());

    // When using CV_TM_CCORR_NORMED, max_loc holds the point with maximum 
    // correlation.
    match_loc = max_loc; 

    // Draw a rectangle in the area that was matched
    cv:rectangle(source, 
                 match_loc, 
                 cv::Point(match_loc.x + template_img.cols , match_loc.y + template_img.rows), 
                 cv::Scalar(255, 0, 0), 2, 8, 0 );

    imshow("Template Match:", source);
    cv::waitKey(0);

    return 0;
}