Got exception boost::python::error_already_set on boost::python::import("cv2")

Question

I got exception boost::python::error_already_set thrown out on calling boost::python::import("cv2")

First, I wrote a python script pymod.py:

import sys
...

and import the pymod.py with boost::python in cpp code:

...
boost::python::import("pymod");
...

works well.

But when I add cv2 in the pymod.py:

import sys, cv2

the boost::python::import("pymod") threw an exception:

terminate called after throwing an instance of 'boost::python::error_already_set'

Notice that the python script with import cv2 can be called by python:

python pymod.py

with no error.

So I tried a cpp version of importing cv2 with boost::python and the same exception is threw out. The code as following:

#include <boost/dll/import.hpp>
#include <boost/python.hpp>

int main(int nArgCnt, char *ppArgs[]) {
    Py_Initialize();
    boost::python::object pySys = boost::python::import("cv2");
    return 0;
}

---

Update 1:

I also have checked the source boost_1_65_1/libs/python/src/import.cpp and found the implementation of boost::python::import:

object BOOST_PYTHON_DECL import(str name)
{
  // should be 'char const *' but older python versions don't use 'const' yet.
  char *n = python::extract<char *>(name);
  python::handle<> module(PyImport_ImportModule(n));
  return python::object(module);
}

And I try to directly use PyImport_ImportModule, but it also failed:

#include <boost/dll/import.hpp>
#include <boost/python.hpp>
#include <Python.h>
#include <iostream>

int main(int nArgCnt, char *ppArgs[]) {
    Py_Initialize();
    // numpy, sys and other modules can be load! VERY STRANGE!!
    auto pMod = PyImport_ImportModule("cv2");
    std::cout << pMod << std::endl;
    return 0;
}

this code print following outputs:

['/usr/local/lib/python3.6/dist-packages', '/usr/local/lib/python3.6/dist-packages/cv2/python-3.6', '/home/devymex/3rdparty/opencv-4.0.1/build/python_loader', '/usr/lib/python36.zip', '/usr/lib/python3.6', '/usr/lib/python3.6/lib-dynload', '/home/devymex/.local/lib/python3.6/site-packages', '/usr/local/lib/python3.6/dist-packages/mxnet-1.3.1-py3.6.egg', '/usr/local/lib/python3.6/dist-packages/graphviz-0.8.4-py3.6.egg', '/usr/local/lib/python3.6/dist-packages/typing-3.6.6-py3.6.egg', '/usr/local/lib/python3.6/dist-packages/typing_extensions-3.6.6-py3.6.egg', '/usr/local/lib/python3.6/dist-packages/onnx-1.5.0-py3.6-linux-x86_64.egg', '/usr/lib/python3/dist-packages']
0

But I'm very sure the command python3 -c "import cv2" works no problem. So it seem irrelevant with boost.

---

Update 2:

PyErr_Print(); gives following information:

Traceback (most recent call last):
  File "/usr/local/lib/python3.6/dist-packages/cv2/__init__.py", line 89, in <module>
    bootstrap()
  File "/usr/local/lib/python3.6/dist-packages/cv2/__init__.py", line 79, in bootstrap
    import cv2
  File "/usr/local/lib/python3.6/dist-packages/cv2/__init__.py", line 89, in <module>
    bootstrap()
  File "/usr/local/lib/python3.6/dist-packages/cv2/__init__.py", line 23, in bootstrap
    raise ImportError('ERROR: recursion is detected during loading of "cv2" binary extensions. Check OpenCV installation.')
ImportError: ERROR: recursion is detected during loading of "cv2" binary extensions. Check OpenCV installation.

__init__.py:

'''
OpenCV Python binary extension loader
'''
import os
import sys

try:
    import numpy
    import numpy.core.multiarray
except ImportError:
    print('OpenCV bindings requires "numpy" package.')
    print('Install it via command:')
    print('    pip install numpy')
    raise

# TODO
# is_x64 = sys.maxsize > 2**32

def bootstrap():
    import sys
    if hasattr(sys, 'OpenCV_LOADER'):
        print(sys.path)
        raise ImportError('ERROR: recursion is detected during loading of "cv2" binary extensions. Check OpenCV installation.')
    sys.OpenCV_LOADER = True

    DEBUG = False
    if hasattr(sys, 'OpenCV_LOADER_DEBUG'):
        DEBUG = True

    import platform
    if DEBUG: print('OpenCV loader: os.name="{}"  platform.system()="{}"'.format(os.name, str(platform.system())))

    LOADER_DIR=os.path.dirname(os.path.abspath(__file__))

    PYTHON_EXTENSIONS_PATHS = []
    BINARIES_PATHS = []

    g_vars = globals()
    l_vars = locals()

    if sys.version_info[:2] < (3, 0):
        from cv2.load_config_py2 import exec_file_wrapper
    else:
        from . load_config_py3 import exec_file_wrapper

    def load_first_config(fnames, required=True):
        for fname in fnames:
            fpath = os.path.join(LOADER_DIR, fname)
            if not os.path.exists(fpath):
                if DEBUG: print('OpenCV loader: config not found, skip: {}'.format(fpath))
                continue
            if DEBUG: print('OpenCV loader: loading config: {}'.format(fpath))
            exec_file_wrapper(fpath, g_vars, l_vars)
            return True
        if required:
            raise ImportError('OpenCV loader: missing configuration file: {}. Check OpenCV installation.'.format(fnames))

    load_first_config(['config.py'], True)
    load_first_config([
        'config-{}.{}.py'.format(sys.version_info[0], sys.version_info[1]),
        'config-{}.py'.format(sys.version_info[0])
    ], True)

    if DEBUG: print('OpenCV loader: PYTHON_EXTENSIONS_PATHS={}'.format(str(l_vars['PYTHON_EXTENSIONS_PATHS'])))
    if DEBUG: print('OpenCV loader: BINARIES_PATHS={}'.format(str(l_vars['BINARIES_PATHS'])))

    for p in reversed(l_vars['PYTHON_EXTENSIONS_PATHS']):
        sys.path.insert(1, p)

    if os.name == 'nt':
        os.environ['PATH'] = ';'.join(l_vars['BINARIES_PATHS']) + ';' + os.environ.get('PATH', '')
        if DEBUG: print('OpenCV loader: PATH={}'.format(str(os.environ['PATH'])))
    else:
        # amending of LD_LIBRARY_PATH works for sub-processes only
        os.environ['LD_LIBRARY_PATH'] = ':'.join(l_vars['BINARIES_PATHS']) + ':' + os.environ.get('LD_LIBRARY_PATH', '')

    if DEBUG: print('OpenCV loader: replacing cv2 module')
    del sys.modules['cv2']
    import cv2

    try:
        import sys
        del sys.OpenCV_LOADER
    except:
        pass

    if DEBUG: print('OpenCV loader: DONE')

bootstrap()

How to solve this problem?

Answer 1

Finally I got the solution:

#include <boost/python/numpy.hpp>
#include <boost/python/extract.hpp>
#include <boost/python.hpp>

#include <iostream>

namespace py = boost::python;

int main(int nArgCnt, char *ppArgs[]) {
    Py_Initialize();
    py::object pySys = py::import("sys");
    auto pySysPaths = py::extract<py::list>(pySys.attr("path"))();

    //Important! Add an empty path to the head of searching paths
    pySysPaths.insert<std::string>(0, "");

    py::object pyCv2 = py::import("cv2");
    return 0;
}

---

Explanation

If you use command python some_script.py, the list of searching path of modules has been modified: a empty path string would be insert into the head of the list:

['', '/usr/local/lib/python3.6/dist-packages', blah, blah, blah, ...]

You can find line 67, 68 of __init__.py of OpenCV:

    for p in reversed(l_vars['PYTHON_EXTENSIONS_PATHS']):
        sys.path.insert(1, p)

Then the path of OpenCV library (cv2.cpython-36m-x86_64-linux-gnu.so) is properly inserted into the list. And import cv2 in line 79 of __init__.py just loading the cv2.cpython-36m-x86_64-linux-gnu.so file directly, everything works well with another searching path up to now.

But things are different in cpp calling. Without the python program, nobody add the empty searching path into head of sys.path, so the __init__.py generated an undeserved list of searching path:

['/usr/local/lib/python3.6/dist-packages', '/usr/local/lib/python3.6/dist-packages/cv2/python-3.6', '/home/devymex/3rdparty/caffe/python', blah, blah, blah, ...]

Notice the real location of /usr/local/lib/python3.6/dist-packages/cv2/python-3.6/cv2.cpython-36m-x86_64-linux-gnu.so is in the second place, so line 79 in __init__.py entered into a recursive calling of __init__.py instead of loading cv2.cpython-36m-x86_64-linux-gnu.so

That's the whole story.