OpenCV image Comparison And Similarity in Android

Question

I'm OpenCV learner. I was trying Image Comparison. I have used OpenCV 2.4.13.3 I have these two images 1.jpg and cam1.jpg.

When I use the following command in openCV

File sdCard = Environment.getExternalStorageDirectory();
String path1, path2;
path1 = sdCard.getAbsolutePath() + "/1.jpg";
path2 = sdCard.getAbsolutePath() + "/cam1.jpg";

FeatureDetector detector = FeatureDetector.create(FeatureDetector.ORB);
DescriptorExtractor extractor = DescriptorExtractor.create(DescriptorExtractor.BRIEF);
DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_HAMMING);

Mat img1 = Highgui.imread(path1);
Mat img2 = Highgui.imread(path2);

Mat descriptors1 = new Mat();
MatOfKeyPoint keypoints1 = new MatOfKeyPoint();
detector.detect(img1, keypoints1);
extractor.compute(img1, keypoints1, descriptors1);

//second image
// Mat img2 = Imgcodecs.imread(path2);
Mat descriptors2 = new Mat();
MatOfKeyPoint keypoints2 = new MatOfKeyPoint();
detector.detect(img2, keypoints2);
extractor.compute(img2, keypoints2, descriptors2);


//matcher image descriptors
MatOfDMatch matches = new MatOfDMatch();
matcher.match(descriptors1,descriptors2,matches);

// Filter matches by distance
MatOfDMatch filtered = filterMatchesByDistance(matches);

int total = (int) matches.size().height;
int Match= (int) filtered.size().height;
Log.d("LOG", "total:" + total + " Match:"+Match);

Method filterMatchesByDistance

    static MatOfDMatch filterMatchesByDistance(MatOfDMatch matches){
    List<DMatch> matches_original = matches.toList();
    List<DMatch> matches_filtered = new ArrayList<DMatch>();

    int DIST_LIMIT = 30;
    // Check all the matches distance and if it passes add to list of filtered matches
    Log.d("DISTFILTER", "ORG SIZE:" + matches_original.size() + "");
    for (int i = 0; i < matches_original.size(); i++) {
        DMatch d = matches_original.get(i);
        if (Math.abs(d.distance) <= DIST_LIMIT) {
            matches_filtered.add(d);
        }
    }
    Log.d("DISTFILTER", "FIL SIZE:" + matches_filtered.size() + "");

    MatOfDMatch mat = new MatOfDMatch();
    mat.fromList(matches_filtered);
    return mat;
}

Log

total:122 Match:30

As we can see from the log match is 30.
But as we can see both images have same visual element (in).
How can I get match=90 using openCV?
It would be great if somebody can help with code snippet.
If using opencv it is not possible then what are the other alternatives we can look for?

Answer 1

But as we can see both images have same visual element (in).

So, we should compare not whole images, but "same visual element" on it. You can improve Match value more if you do not compare the "template" and "camera" images themselves, but processed same way (converted to binary black/white, for example) "template" and "camera" images. For example, try to find blue (background of template logo) square on both ("template" and "camera") images and compare that squares (Region Of Interest). The code may be something like that:

Bitmap bmImageTemplate = <get your template image Bitmap>;
Bitmap bmTemplate = findLogo(bmImageTemplate);  // process template image

Bitmap bmImage = <get your camera image Bitmap>;
Bitmap bmLogo = findLogo(bmImage); // process camera image same way

compareBitmaps(bmTemplate, bmLogo);

where

private Bitmap findLogo(Bitmap sourceBitmap) {
    Bitmap roiBitmap = null;
    Mat sourceMat = new Mat(sourceBitmap.getWidth(), sourceBitmap.getHeight(), CvType.CV_8UC3);
    Utils.bitmapToMat(sourceBitmap, sourceMat);
    Mat roiTmp = sourceMat.clone();

    final Mat hsvMat = new Mat();
    sourceMat.copyTo(hsvMat);

    // convert mat to HSV format for Core.inRange()
    Imgproc.cvtColor(hsvMat, hsvMat, Imgproc.COLOR_RGB2HSV);

    Scalar lowerb = new Scalar(85, 50, 40);         // lower color border for BLUE
    Scalar upperb = new Scalar(135, 255, 255);      // upper color border for BLUE
    Core.inRange(hsvMat, lowerb, upperb, roiTmp);   // select only blue pixels

    // find contours
    List<MatOfPoint> contours = new ArrayList<>();
    List<Rect> squares = new ArrayList<>();
    Imgproc.findContours(roiTmp, contours, new Mat(), Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);

    // find appropriate bounding rectangles
    for (MatOfPoint contour : contours) {
        MatOfPoint2f areaPoints = new MatOfPoint2f(contour.toArray());
        RotatedRect boundingRect = Imgproc.minAreaRect(areaPoints);

        double rectangleArea = boundingRect.size.area();

        // test min ROI area in pixels
        if (rectangleArea > 400) {
            Point rotated_rect_points[] = new Point[4];
            boundingRect.points(rotated_rect_points);

            Rect rect = Imgproc.boundingRect(new MatOfPoint(rotated_rect_points));
            double aspectRatio = rect.width > rect.height ?
                    (double) rect.height / (double) rect.width : (double) rect.width / (double) rect.height;
            if (aspectRatio >= 0.9) {
                squares.add(rect);
            }
        }
    }

    Mat logoMat = extractSquareMat(roiTmp, getBiggestSquare(squares));

    roiBitmap = Bitmap.createBitmap(logoMat.cols(), logoMat.rows(), Bitmap.Config.ARGB_8888);
    Utils.matToBitmap(logoMat, roiBitmap);
    return roiBitmap;
}

method extractSquareMat() just extract Region of Interest (logo) from whole image

public static Mat extractSquareMat(Mat sourceMat, Rect rect) {
    Mat squareMat = null;
    int padding = 50;

    if (rect != null) {
        Rect truncatedRect = new Rect((int) rect.tl().x + padding, (int) rect.tl().y + padding,
                rect.width - 2 * padding, rect.height - 2 * padding);
        squareMat = new Mat(sourceMat, truncatedRect);
    }

    return squareMat ;
}

and compareBitmaps() just wrapper for your code:

private void compareBitmaps(Bitmap bitmap1, Bitmap bitmap2) {
    Mat mat1 = new Mat(bitmap1.getWidth(), bitmap1.getHeight(), CvType.CV_8UC3);
    Utils.bitmapToMat(bitmap1, mat1);

    Mat mat2 = new Mat(bitmap2.getWidth(), bitmap2.getHeight(), CvType.CV_8UC3);
    Utils.bitmapToMat(bitmap2, mat2);

    compareMats(mat1, mat2);
}

your code as method:

private void compareMats(Mat img1, Mat img2) {
    FeatureDetector detector = FeatureDetector.create(FeatureDetector.ORB);
    DescriptorExtractor extractor = DescriptorExtractor.create(DescriptorExtractor.BRIEF);
    DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_HAMMING);

    Mat descriptors1 = new Mat();
    MatOfKeyPoint keypoints1 = new MatOfKeyPoint();
    detector.detect(img1, keypoints1);
    extractor.compute(img1, keypoints1, descriptors1);

    //second image
    // Mat img2 = Imgcodecs.imread(path2);
    Mat descriptors2 = new Mat();
    MatOfKeyPoint keypoints2 = new MatOfKeyPoint();
    detector.detect(img2, keypoints2);
    extractor.compute(img2, keypoints2, descriptors2);


    //matcher image descriptors
    MatOfDMatch matches = new MatOfDMatch();
    matcher.match(descriptors1,descriptors2,matches);

    // Filter matches by distance
    MatOfDMatch filtered = filterMatchesByDistance(matches);

    int total = (int) matches.size().height;
    int Match= (int) filtered.size().height;
    Log.d("LOG", "total:" + total + " Match:" + Match);
}

static MatOfDMatch filterMatchesByDistance(MatOfDMatch matches){
    List<DMatch> matches_original = matches.toList();
    List<DMatch> matches_filtered = new ArrayList<DMatch>();

    int DIST_LIMIT = 30;
    // Check all the matches distance and if it passes add to list of filtered matches
    Log.d("DISTFILTER", "ORG SIZE:" + matches_original.size() + "");
    for (int i = 0; i < matches_original.size(); i++) {
        DMatch d = matches_original.get(i);
        if (Math.abs(d.distance) <= DIST_LIMIT) {
            matches_filtered.add(d);
        }
    }
    Log.d("DISTFILTER", "FIL SIZE:" + matches_filtered.size() + "");

    MatOfDMatch mat = new MatOfDMatch();
    mat.fromList(matches_filtered);
    return mat;
}

As result for resized (scaled for 50%) images saved from your question result is:

D/DISTFILTER: ORG SIZE:237
D/DISTFILTER: FIL SIZE:230
D/LOG: total:237 Match:230

NB! This is a quick and dirty example just to demonstrate approach only for given template.

P.S. getBiggestSquare() can be like that (based on compare by area):

public static Rect getBiggestSquare(List<Rect> squares) {
    Rect biggestSquare = null;

    if (squares != null && squares.size() >= 1) {
        Rect square;
        double maxArea;
        int ixMaxArea = 0;

        square = squares.get(ixMaxArea);
        maxArea = square.area();

        for (int ix = 1; ix < squares.size(); ix++) {
            square = squares.get(ix);
            if (square.area() > maxArea) {
                maxArea = square.area();
                ixMaxArea = ix;
            }
        }

        biggestSquare = squares.get(ixMaxArea);
    }

    return biggestSquare;
}