What is the easiest way to convert ndarray into cv::Mat?

Question

I'm trying to create Python/Cython wrapper for C++ library that uses cv::Mat class from OpenCV. In official Python wrapper all functions take NumPy's ndarray instead of cv::Mat, which is quite convenient. But in my own wrapper, how do I do such conversion? That is, how do I create cv::Mat from np.ndarray?

Answer 1

As suggested by kyamagu, you can use OpenCV's official python wrapper code, especially the pyopencv_to and pyopencv_from.

I've been struggling as you did with all the dependencies and the generated header files. Nevertheless, it is possible to reduce the complexity of this by "cleaning" the cv2.cpp as lightalchemist did here in order to keep only what is necessary. You will need to adapt it to your need and to the version of OpenCV you're using but its basically the same code that I used.

#include <Python.h>
#include "numpy/ndarrayobject.h"
#include "opencv2/core/core.hpp"

static PyObject* opencv_error = 0;

static int failmsg(const char *fmt, ...)
{
    char str[1000];

    va_list ap;
    va_start(ap, fmt);
    vsnprintf(str, sizeof(str), fmt, ap);
    va_end(ap);

    PyErr_SetString(PyExc_TypeError, str);
    return 0;
}

class PyAllowThreads
{
public:
    PyAllowThreads() : _state(PyEval_SaveThread()) {}
    ~PyAllowThreads()
    {
        PyEval_RestoreThread(_state);
    }
private:
    PyThreadState* _state;
};

class PyEnsureGIL
{
public:
    PyEnsureGIL() : _state(PyGILState_Ensure()) {}
    ~PyEnsureGIL()
    {
        PyGILState_Release(_state);
    }
private:
    PyGILState_STATE _state;
};

#define ERRWRAP2(expr) \
try \
{ \
    PyAllowThreads allowThreads; \
    expr; \
} \
catch (const cv::Exception &e) \
{ \
    PyErr_SetString(opencv_error, e.what()); \
    return 0; \
}

using namespace cv;

static PyObject* failmsgp(const char *fmt, ...)
{
  char str[1000];

  va_list ap;
  va_start(ap, fmt);
  vsnprintf(str, sizeof(str), fmt, ap);
  va_end(ap);

  PyErr_SetString(PyExc_TypeError, str);
  return 0;
}

static size_t REFCOUNT_OFFSET = (size_t)&(((PyObject*)0)->ob_refcnt) +
    (0x12345678 != *(const size_t*)"\x78\x56\x34\x12\0\0\0\0\0")*sizeof(int);

static inline PyObject* pyObjectFromRefcount(const int* refcount)
{
    return (PyObject*)((size_t)refcount - REFCOUNT_OFFSET);
}

static inline int* refcountFromPyObject(const PyObject* obj)
{
    return (int*)((size_t)obj + REFCOUNT_OFFSET);
}

class NumpyAllocator : public MatAllocator
{
public:
    NumpyAllocator() {}
    ~NumpyAllocator() {}

    void allocate(int dims, const int* sizes, int type, int*& refcount,
                  uchar*& datastart, uchar*& data, size_t* step)
    {
        PyEnsureGIL gil;

        int depth = CV_MAT_DEPTH(type);
        int cn = CV_MAT_CN(type);
        const int f = (int)(sizeof(size_t)/8);
        int typenum = depth == CV_8U ? NPY_UBYTE : depth == CV_8S ? NPY_BYTE :
                      depth == CV_16U ? NPY_USHORT : depth == CV_16S ? NPY_SHORT :
                      depth == CV_32S ? NPY_INT : depth == CV_32F ? NPY_FLOAT :
                      depth == CV_64F ? NPY_DOUBLE : f*NPY_ULONGLONG + (f^1)*NPY_UINT;
        int i;
        npy_intp _sizes[CV_MAX_DIM+1];
        for( i = 0; i < dims; i++ )
            _sizes[i] = sizes[i];
        if( cn > 1 )
        {
            /*if( _sizes[dims-1] == 1 )
                _sizes[dims-1] = cn;
            else*/
                _sizes[dims++] = cn;
        }
        PyObject* o = PyArray_SimpleNew(dims, _sizes, typenum);
        if(!o)
            CV_Error_(CV_StsError, ("The numpy array of typenum=%d, ndims=%d can not be created", typenum, dims));
        refcount = refcountFromPyObject(o);
        npy_intp* _strides = PyArray_STRIDES(o);
        for( i = 0; i < dims - (cn > 1); i++ )
            step[i] = (size_t)_strides[i];
        datastart = data = (uchar*)PyArray_DATA(o);
    }

    void deallocate(int* refcount, uchar*, uchar*)
    {
        PyEnsureGIL gil;
        if( !refcount )
            return;
        PyObject* o = pyObjectFromRefcount(refcount);
        Py_INCREF(o);
        Py_DECREF(o);
    }
};

NumpyAllocator g_numpyAllocator;

enum { ARG_NONE = 0, ARG_MAT = 1, ARG_SCALAR = 2 };

static int pyopencv_to(const PyObject* o, Mat& m, const char* name = "<unknown>", bool allowND=true)
{
    if(!o || o == Py_None)
    {
        if( !m.data )
            m.allocator = &g_numpyAllocator;
        return true;
    }

    if( PyInt_Check(o) )
    {
        double v[] = {PyInt_AsLong((PyObject*)o), 0., 0., 0.};
        m = Mat(4, 1, CV_64F, v).clone();
        return true;
    }
    if( PyFloat_Check(o) )
    {
        double v[] = {PyFloat_AsDouble((PyObject*)o), 0., 0., 0.};
        m = Mat(4, 1, CV_64F, v).clone();
        return true;
    }
    if( PyTuple_Check(o) )
    {
        int i, sz = (int)PyTuple_Size((PyObject*)o);
        m = Mat(sz, 1, CV_64F);
        for( i = 0; i < sz; i++ )
        {
            PyObject* oi = PyTuple_GET_ITEM(o, i);
            if( PyInt_Check(oi) )
                m.at<double>(i) = (double)PyInt_AsLong(oi);
            else if( PyFloat_Check(oi) )
                m.at<double>(i) = (double)PyFloat_AsDouble(oi);
            else
            {
                failmsg("%s is not a numerical tuple", name);
                m.release();
                return false;
            }
        }
        return true;
    }

    if( !PyArray_Check(o) )
    {
        failmsg("%s is not a numpy array, neither a scalar", name);
        return false;
    }

    bool needcopy = false, needcast = false;
    int typenum = PyArray_TYPE(o), new_typenum = typenum;
    int type = typenum == NPY_UBYTE ? CV_8U :
               typenum == NPY_BYTE ? CV_8S :
               typenum == NPY_USHORT ? CV_16U :
               typenum == NPY_SHORT ? CV_16S :
               typenum == NPY_INT ? CV_32S :
               typenum == NPY_INT32 ? CV_32S :
               typenum == NPY_FLOAT ? CV_32F :
               typenum == NPY_DOUBLE ? CV_64F : -1;

    if( type < 0 )
    {
        if( typenum == NPY_INT64 || typenum == NPY_UINT64 || type == NPY_LONG )
        {
            needcopy = needcast = true;
            new_typenum = NPY_INT;
            type = CV_32S;
        }
        else
        {
            failmsg("%s data type = %d is not supported", name, typenum);
            return false;
        }
    }

    int ndims = PyArray_NDIM(o);
    if(ndims >= CV_MAX_DIM)
    {
        failmsg("%s dimensionality (=%d) is too high", name, ndims);
        return false;
    }

    int size[CV_MAX_DIM+1];
    size_t step[CV_MAX_DIM+1], elemsize = CV_ELEM_SIZE1(type);
    const npy_intp* _sizes = PyArray_DIMS(o);
    const npy_intp* _strides = PyArray_STRIDES(o);
    bool ismultichannel = ndims == 3 && _sizes[2] <= CV_CN_MAX;

    for( int i = ndims-1; i >= 0 && !needcopy; i-- )
    {
        // these checks handle cases of
        //  a) multi-dimensional (ndims > 2) arrays, as well as simpler 1- and 2-dimensional cases
        //  b) transposed arrays, where _strides[] elements go in non-descending order
        //  c) flipped arrays, where some of _strides[] elements are negative
        if( (i == ndims-1 && (size_t)_strides[i] != elemsize) ||
            (i < ndims-1 && _strides[i] < _strides[i+1]) )
            needcopy = true;
    }

    if( ismultichannel && _strides[1] != (npy_intp)elemsize*_sizes[2] )
        needcopy = true;

    if (needcopy)
    {
        if( needcast )
            o = (PyObject*)PyArray_Cast((PyArrayObject*)o, new_typenum);
        else
            o = (PyObject*)PyArray_GETCONTIGUOUS((PyArrayObject*)o);
        _strides = PyArray_STRIDES(o);
    }

    for(int i = 0; i < ndims; i++)
    {
        size[i] = (int)_sizes[i];
        step[i] = (size_t)_strides[i];
    }

    // handle degenerate case
    if( ndims == 0) {
        size[ndims] = 1;
        step[ndims] = elemsize;
        ndims++;
    }

    if( ismultichannel )
    {
        ndims--;
        type |= CV_MAKETYPE(0, size[2]);
    }

    if( ndims > 2 && !allowND )
    {
        failmsg("%s has more than 2 dimensions", name);
        return false;
    }

    m = Mat(ndims, size, type, PyArray_DATA(o), step);

    if( m.data )
    {
        m.refcount = refcountFromPyObject(o);
        if (!needcopy)
        {
            m.addref(); // protect the original numpy array from deallocation
                        // (since Mat destructor will decrement the reference counter)
        }
    };
    m.allocator = &g_numpyAllocator;

    return true;
}

static PyObject* pyopencv_from(const Mat& m)
{
    if( !m.data )
        Py_RETURN_NONE;
    Mat temp, *p = (Mat*)&m;
    if(!p->refcount || p->allocator != &g_numpyAllocator)
    {
        temp.allocator = &g_numpyAllocator;
        ERRWRAP2(m.copyTo(temp));
        p = &temp;
    }
    p->addref();
    return pyObjectFromRefcount(p->refcount);
}

Once you have a cleaned up cv2.cpp file, here is some Cython code that takes care of the conversion. Notice the definition and the call to the import_array() function (it's a NumPy function defined in a header included somewhere in cv2.cpp), this is necessary to define some macros used by pyopencv_to, if you don't call it you will get segmentation faults as lightalchemist pointed out.

from cpython.ref cimport PyObject

# Declares OpenCV's cv::Mat class
cdef extern from "opencv2/core/core.hpp":
    cdef cppclass Mat:
        pass

# Declares the official wrapper conversion functions + NumPy's import_array() function
cdef extern from "cv2.cpp":
    void import_array()
    PyObject* pyopencv_from(const _Mat&)
    int pyopencv_to(PyObject*, _Mat&)


# Function to be called at initialization
cdef void init():
    import_array()

# Python to C++ conversion
cdef Mat nparrayToMat(object array):
    cdef Mat mat
    cdef PyObject* pyobject = <PyObject*> array
    pyopencv_to(pyobject, mat)
    return <Mat> mat

# C++ to Python conversion
cdef object matToNparray(Mat mat):
    return <object> pyopencv_from(mat)

Note: somehow I got an error with NumPy 1.8.0 on Fedora 20 while compiling due to a strange return statement in the import_array macro, I had to manually remove it to make it work but I can't find this return statement in the NumPy's 1.8.0 GitHub source code

Answer 2

It turns out that there's no simple way to convert (any) np.ndarray into corresponding cv::Mat. Basically, one needs to do only 2 things:

1. Create empty cv::Mat of corresponding size and type.
2. Copy data.

However, devil hides in details. Both ndarray and Mat may hold quite varying data formats. For instance, data in NumPy arrays may be in C or in Fortran order, array object may own its data or keep a view to another array, channels may go in a different order (RGB in NumPy vs. BGR in OpenCV), etc.

So instead of trying to solve generic problem I decided to stay with simple code that fits my needs and may be easily modified by anyone interested.

Following code in Cython works with float32/CV_32FC1 images with default byte order:

cdef void array2mat(np.ndarray arr, Mat& mat):
    cdef int r = arr.shape[0]
    cdef int c = arr.shape[1]
    cdef int mat_type = CV_32FC1            # or CV_64FC1, or CV_8UC3, or whatever
    mat.create(r, c, mat_type)
    cdef unsigned int px_size = 4           # 8 for single-channel double image or 
                                            #   1*3 for three-channel uint8 image
    memcpy(mat.data, arr.data, r*c*px_size)

To use this code in Cython one also needs to declare some types and constants, e.g. like this:

import numpy as np
# Cython makes it simple to import NumPy
cimport numpy as np


# OpenCV's matrix class
cdef extern from "opencv2/opencv.hpp" namespace "cv":

    cdef cppclass Mat:
        Mat() except +
        Mat(int, int, int, void*) except +
    void create(int, int, int)
        void* data
        int type() const
        int cols
        int rows
        int channels()
        Mat clone() const

# some OpenCV matrix types
cdef extern from "opencv2/opencv.hpp":        
    cdef int CV_8UC3
    cdef int CV_8UC1
    cdef int CV_32FC1
    cdef int CV_64FC1

Opposite conversion (from cv::Mat to np.ndarray) may be achieved in a similar way.

Bonus: there's also nice blog post describing same kind of conversion for RGB/BGR images.