How to extract lines from each contour in OpenCV for Android?

Question

I'd like to examine each Canny detected edge and look for the main lines in it (to check if they seem to shape a rectangle, for example if 2 pairs of lines are parallel etc.).

Imgproc.HoughLinesP does what I want, but it gives the lines from the whole image, and I want to know which lines come from the same edges.

I tried also FindContours, and looking for main lines in each contour with approxPolyDP, but this doesn't look adapted because there are often gaps in Canny detected edges. This gives contours of the edges and not the edges themselves.

Here is a test image example :

How can I get a set of lines for each shape ?

Answer 1

Based on Miki's answer, here is what I've done :

• Canny
• HoughLinesP (or LineSegmentDetector, as you want) : to detect lines
• ConnectedComponents : to find Canny "contours" in the Canny image.
• Dilate with a 3x3 kernel (see below)
• For each Hough line : take a few pixels from the line and look for the most frequent value (ignore 0's). For example, I chose {p1 , 0.75*p1 + 0.25*p2, 0.5*p1 + 0.5*p2, 0.25*p1 + 0.75*p2, p2}, so if my values are {1,2,0,2,2} then the line belongs to the connectedComponent number 2. Dilating is to be sure you didn't miss a contour by only 1 pixel (but don't use it if your objects are too close).

This allows to "tag" HoughLines with the color of the contour they belong to.

All of these functions can be found in Imgproc module, this works in OpenCV 3.0 only and gives the desired result.

Here is a code :

// open image
File root = Environment.getExternalStorageDirectory();
File file = new File(root, "image_test.png");

Mat mRGBA = Imgcodecs.imread(file.getAbsolutePath());
Imgproc.cvtColor(mRGBA, mRGBA,  Imgproc.COLOR_BGR2RGB);

Mat mGray = new Mat();
Imgproc.cvtColor(mRGBA, mGray, Imgproc.COLOR_RGBA2GRAY);

Imgproc.medianBlur(mGray, mGray, 7);

/* Main part */

Imgproc.Canny(mGray, mGray, 50, 60, 3, true);

Mat aretes = new Mat();
Imgproc.HoughLinesP(mGray, aretes, 1, 0.01745329251, 30, 10, 4);

/**
 * Tag Canny edges in the gray picture with indexes from 1 to 65535 (0 = background)
 * (Make sure there are less than 255 components or convert mGray to 16U before)
 */
int nb = Imgproc.connectedComponents(mGray,mGray,8,CvType.CV_16U);

Imgproc.dilate(mGray, mGray, Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(3,3)));


// for each Hough line
for (int x = 0; x < aretes.rows(); x++) {
     double[] vec = aretes.get(x, 0);
     double x1 = vec[0],
            y1 = vec[1],
            x2 = vec[2],
            y2 = vec[3];

     /**
      * Take 5 points from the line
      *
      *   x----x----x----x----x
      *   P1                  P2
      */
        double[] pixel_values = new double[5];
        pixel_values[0] = mGray.get((int) y1, (int) x1)[0];
        pixel_values[1] = mGray.get((int) (y1*0.75 + y2*0.25), (int) (x1*0.75 + x2*0.25))[0];
        pixel_values[2] = mGray.get((int) ((y1 + y2) *0.5), (int) ((x1 + x2) *0.5))[0];
        pixel_values[3] = mGray.get((int) (y1*0.25 + y2*0.75), (int) (x1*0.25 + x2*0.75))[0];
        pixel_values[4] = mGray.get((int) y2, (int) x2)[0];

        /**
         * Look for the most frequent value
         * (To make it readable, the following code accepts the line only if there are at
         * least 3 good pixels)
         */
        double value;
        Arrays.sort(pixel_values);

        if (pixel_values[1] == pixel_values[3] || pixel_values[0] == pixel_values[2] || pixel_values[2] == pixel_values[4]) {
            value = pixel_values[2];
        }
        else {
            value = 0;
        }

        /**
         * Now value is the index of the connected component (or 0 if it's a bad line)
         * You can store it in an other array, here I'll just draw the line with the value
         */
        if (value != 0) {
            Imgproc.line(mRGBA,new Point(x1,y1),new Point(x2,y2),new Scalar((value * 41 + 50) % 255, (value * 69 + 100) % 255, (value * 91 + 60) % 255),3);
        }
}

Imgproc.cvtColor(mRGBA, mRGBA, Imgproc.COLOR_RGB2BGR);
File file2 = new File(root, "image_test_OUT.png");
Imgcodecs.imwrite(file2.getAbsolutePath(), mRGBA);

Answer 2

If you're using OpenCV 3.0.0 you can use LineSegmentDetector, and "AND" your detected lines with the contours.

I provide a sample code below. It's C++ (sorry about that), but you can easily translate in Java. At least you see how to use LineSegmentDetector and how extract common lines for each contour. You'll see the lines on the same contour with the same color.

#include <opencv2/opencv.hpp>
using namespace cv;
using namespace std;

int main()
{
    RNG rng(12345);
    Mat3b img = imread("path_to_image");
    Mat1b gray;
    cvtColor(img, gray, COLOR_BGR2GRAY);

    Mat3b result;
    cvtColor(gray, result, COLOR_GRAY2BGR);

    // Detect lines
    Ptr<LineSegmentDetector> detector = createLineSegmentDetector();
    vector<Vec4i> lines;
    detector->detect(gray, lines);

    // Draw lines
    Mat1b lineMask(gray.size(), uchar(0));
    for (int i = 0; i < lines.size(); ++i)
    {
        line(lineMask, Point(lines[i][0], lines[i][1]), Point(lines[i][2], lines[i][3]), Scalar(255), 2);
    }

    // Compute edges
    Mat1b edges;
    Canny(gray, edges, 200, 400);

    // Find contours
    vector<vector<Point>> contours;
    findContours(edges.clone(), contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE);

    for (int i = 0; i < contours.size(); ++i)
    {
        // Draw each contour
        Mat1b contourMask(gray.size(), uchar(0));
        drawContours(contourMask, contours, i, Scalar(255), 2); // Better use 1 here. 2 is just for visualization purposes

        // AND the contour and the lines
        Mat1b bor;
        bitwise_and(contourMask, lineMask, bor);

        // Draw the common pixels with a random color
        vector<Point> common;
        findNonZero(bor, common);

        Vec3b color = Vec3b(rng.uniform(0, 255), rng.uniform(0, 255), rng.uniform(0, 255));
        for (int j = 0; j < common.size(); ++j)
        {
            result(common[j]) = color;
        }
    }


    imshow("result", result);
    waitKey();

    return 0;
}