I have written an extension module that uses C++ function pointers to store sequences of function calls. I want to 'run' these call sequences in separate processes using python's multiprocessing
module (there's no shared state, so no synchronization issues).
I need to know if function pointers (not data pointers) remain valid after multiprocessing
does it's fork()
.
C++ module:
#include <list>
#include <boost/assert.hpp>
#include <boost/python.hpp>
#include <boost/python/stl_iterator.hpp>
#include <boost/foreach.hpp>
/*
* Some functions to be called
*/
double funcA(double d) { return d; }
double funcB(double d) { return d + 3.14; }
double funcC(double d) { return d - 42.0; }
/*
* My container of function pointers (picklable to allow use with multiprocessing)
*/
typedef double(*func_ptr_t)(double);
struct CallSequence {
CallSequence() {
_seq.push_back(funcA);
_seq.push_back(funcB);
_seq.push_back(funcC);
}
std::list<func_ptr_t> _seq;
};
template <typename cast_type>
struct CallSequence_picklesuite : boost::python::pickle_suite {
BOOST_STATIC_ASSERT_MSG(sizeof(cast_type) == sizeof(func_ptr_t), CANNOT_CAST_POINTER_TO_REQUESTED_TYPE);
static boost::python::list getstate(const CallSequence& cs) {
boost::python::list ret;
BOOST_FOREACH(func_ptr_t p, cs._seq)
ret.append(reinterpret_cast<cast_type>(p));
return ret;
}
static void setstate(CallSequence& cs, boost::python::list l) {
std::list<func_ptr_t> new_list;
boost::python::stl_input_iterator<cast_type> begin(l), end;
for(; begin != end; begin++)
new_list.push_back(reinterpret_cast<func_ptr_t>(*begin));
cs._seq.swap(new_list);
}
};
/*
* Run the call sequence
*/
double runner(const CallSequence& cs) {
double ret = 0;
BOOST_FOREACH(const func_ptr_t& p, cs._seq)
ret += p(2.18);
return ret;
}
BOOST_PYTHON_MODULE(my_extension) {
using namespace ::boost::python;
class_<CallSequence>("CallSequence")
.def_pickle(CallSequence_picklesuite<unsigned int>());
def("runner", runner);
}
Compiled with:
$ g++ question1.cpp -lboost_python -I /usr/include/python2.7 -shared -o my_extension.so
Python code invoking it across multiple processes:
#!/usr/bin/python
from multiprocessing import Pool
import my_extension
def runner(sequence):
return my_extension.runner(sequence)
def main():
l = [my_extension.CallSequence() for _ in range(200)]
pool = Pool(processes=4)
print pool.map(runner, l)
if __name__ == '__main__':
main()
The output is as expected. I want to know if I'm just 'getting lucky' or if I can reliably expect function pointers to remain valid after a fork()
.
Sure--the address space is copied when you fork, so the pointers are still valid afterward for both parent and child processes.
If you love us? You can donate to us via Paypal or buy me a coffee so we can maintain and grow! Thank you!
Donate Us With