Apply a Python function to an std::vector via Cython (callback)

Question

I am trying to learn how to use callbacks between C and Python by way of Cython and have been looking at this demo. I would like a Python function applied to one std::vector/numpy.array and store the results in another. I can compile and run without errors, but ultimately the vector y is not being changed.

C++ header

// callback.hpp
#include<vector> 
typedef double (*Callback)( void *apply, double &x );
void function( Callback callback, void *apply, vector<double> &x, 
               vector<double> &y );

C++ source

// callback.cpp
#include "callback.hpp"
#include <iostream>
using namespace std;

void function( Callback callback,  void* apply,
               vector<double> &x,  vector<double> &y ) {

int n = x.size();

for(int i=0;i<n;++i) {
    y[i] = callback(apply,x[i]);
    std::cout << y[i] << std::endl;
}

Cython header

# cy_callback.pxd
import cython
from libcpp.vector cimport vector

cdef extern from "callback.hpp":
     ctypedef double (*Callback)( void *apply, double &x )
void function( Callback callback, void* apply, vector[double] &x,
               vector[double] &y )

Cython source

# cy_callback.pyx
from cy_callback cimport function
from libcpp.vector cimport vector

def pyfun(f,x,y):
    function( cb, <void*> f, <vector[double]&> x, <vector[double]&> y )

cdef double cb(void* f, double &x):
    return (<object>f)(x)

I compile with fairly boilerplate setup: python setup.py build_ext -i

# setup.py
from distutils.core import setup
from distutils.extension import Extension
from Cython.Distutils import build_ext
import numpy
import os
os.environ["CC"] = "g++"
os.environ["CXX"] = "g++"
setup( name = 'callback',
ext_modules=[Extension("callback",
             sources=["cy_callback.pyx","callback.cpp"],
             language="c++",
             include_dirs=[numpy.get_include()])],
cmdclass = {'build_ext': build_ext},
) 

And finally test with the Python script

# test.py
import numpy as np
from callback import pyfun

x = np.arange(11)
y = np.zeros(11)
pyfun(lambda x:x**2,x,y)
print(y)

When the elements of y are set in callback.cpp, the correct values are being printed to the screen, which means that pyfun is indeed being evaluated correctly, however, at the Python level, y remains all zeros.

Any idea what I'm doing incorrectly?

Answer 1

Thanks, Ian. Based on your suggestion, I changed the code to return a vector instead of trying to modify it in place. This works, although is admittedly not particularly efficient

callback.hpp

 typedef double (*Callback)( void *apply, double x );

 vector<double> function( Callback callback, void *apply,
                          const vector<double> &x );

callback.cpp

vector<double> function( Callback callback, void* apply, 
                     const vector<double> &x ) {

    int n = x.size();
    vector<double> y(n);

    for(int i=0;i<n;++i) {
        y[i] = callback(apply,x[i]);
    }
    return y;
}

cy_callback.pxd

cdef extern from "callback.hpp":
    ctypedef double (*Callback)( void *apply, const double &x )

vector[double] function( Callback callback, void* apply, 
                         vector[double] &x )

cy_callback.pyx

from cy_callback cimport function
from libcpp.vector cimport vector

def pyfun(f,x):
    return function( cb, <void*> f, <const vector[double]&> x )


cdef double cb(void* f, double x ):
    return (<object>f)(x)

test.py

import numpy as np
from callback import pyfun

x = np.arange(11)
y = np.zeros(11)

def f(x):
    return x*x  

y = pyfun(f,x)
print(y)

Answer 2

There is an implicit copy of the array made when you cast it to be a vector. There isn't currently any way to have a vector take ownership of memory that has already been allocated, so the only workaround will be to copy the values manually or by exposing std::copy to cython. See How to cheaply assign C-style array to std::vector?.