Clean way of structuring ctypes class

Question

I've defined a ctypes class and an associated convenience function like so:

class BNG_FFITuple(Structure):     _fields_ = [("a", c_uint32),                 ("b", c_uint32)]   class BNG_FFIArray(Structure):     _fields_ = [("data", c_void_p),                 ("len", c_size_t)]      # Allow implicit conversions from a sequence of 32-bit unsigned ints     @classmethod     def from_param(cls, seq):         return seq if isinstance(seq, cls) else cls(seq)      def __init__(self, seq, data_type = c_float):         array_type = data_type * len(seq)         raw_seq = array_type(*seq)         self.data = cast(raw_seq, c_void_p)         self.len = len(seq)   def bng_void_array_to_tuple_list(array, _func, _args):     res = cast(array.data, POINTER(BNG_FFITuple * array.len))[0]     return res  convert = lib.convert_to_bng convert.argtypes = (BNG_FFIArray, BNG_FFIArray) convert.restype = BNG_FFIArray convert.errcheck = bng_void_array_to_tuple_list drop_array = lib.drop_array  drop_array.argtypes = (POINTER(BNG_FFIArray),) 

I then define a simple convenience function:

def f(a, b):     return [(i.a, i.b) for i in iter(convert(a, b))] 

Most of this works perfectly, but I have two issues:

• It's not flexible enough; I'd like to be able to instantiate a BNG_FFITuple using c_float instead of c_uint32 (so the fields are c_float), and vice versa, so the BNG_FFIArray data_type is c_uint32. I'm not clear on how to do this, though.
• I'd like to free the memory which is now owned by Python, by sending a POINTER(BNG_FFIArray) back to my dylib (see drop_array – I've already defined a function in my dylib), but I'm not sure at what point I should call it.

Is there a way of encapsulating all this in a neater, more Pythonic way, which is also safer? I'm concerned that without the memory cleanup being defined in a robust way (on __exit__? __del__?) That anything that goes wrong will lead to unfreed memory

Answer 1

Since you have some control over the rust side, the cleanest thing to do would be to pre-allocate the result array from Python before the call, and pass everything in a single structure.

The code below assumes this modification, but also designates the place where you would do the deallocation if you cannot do this.

Note that if you do this sort of encapsulation, you do NOT need to specify things like the parameters and result processing for the library function, because you're only calling the actual function from a single place, and always with exactly the same kinds of parameters.

I don't know rust (and even my C is a bit rusty), but the code below assumes you redefine your rust to match the equivalent of something like this:

typedef struct FFIParams {     int32 source_ints;     int32 len;     void * a;     void * b;     void * result; } FFIParams;  void convert_to_bng(FFIParams *p) { } 

Here is the Python. One final note -- this is not thread-safe, due to the reuse of the parameter structure. That's easy enough to fix if needed.

from ctypes import c_uint32, c_float, c_size_t, c_void_p from ctypes import Structure, POINTER, pointer, cast from itertools import izip, islice  _test_standalone = __name__ == '__main__'  if _test_standalone:     class lib(object):         @staticmethod         def convert_to_bng(ptr_params):             params = ptr_params.contents             source_ints = params.source_ints             types = c_uint32, c_float             if not source_ints:                 types = reversed(types)             length = params.len             src_type, dst_type = types             src_type = POINTER(length * src_type)             dst_type = POINTER(length * 2 * dst_type)             a = cast(params.a, src_type).contents             b = cast(params.b, src_type).contents             result = cast(params.result, dst_type).contents              # Assumes we are converting int to float or back...             func = float if source_ints else int             result[0::2] = map(func, a)             result[1::2] = map(func, b)  class _BNG_FFIParams(Structure):     _fields_ = [("source_ints", c_uint32),                 ("len", c_size_t),                 ("a", c_void_p),                 ("b", c_void_p),                 ("result", c_void_p)]  class _BNG_FFI(object):      int_type = c_uint32     float_type = c_float     _array_type = type(10 * int_type)      # This assumes we want the result to be opposite type.     # Maybe I misunderstood this -- easily fixable if so.     _result_type = {int_type: float_type, float_type: int_type}      def __init__(self):         my_params = _BNG_FFIParams()         self._params = my_params         self._pointer = POINTER(_BNG_FFIParams)(my_params)         self._converter = lib.convert_to_bng       def _getarray(self, seq, data_type):         # Optimization for pre-allocated correct array type         if type(type(seq)) == self._array_type and seq._type_ is data_type:             print("Optimized!")             return seq         return (data_type * len(seq))(*seq)      def __call__(self, a, b, data_type=float_type):         length = len(a)         if length != len(b):             raise ValueError("Input lengths must be same")          a, b = (self._getarray(x, data_type) for x in (a, b))          # This has the salutary side-effect of insuring we were         # passed a valid type         result = (length * 2 * self._result_type[data_type])()          params = self._params         params.source_ints = data_type is self.int_type         params.len = length         params.a = cast(pointer(a), c_void_p)         params.b = cast(pointer(b), c_void_p)         params.result = cast(pointer(result), c_void_p)         self._converter(self._pointer)          evens = islice(result, 0, None, 2)         odds = islice(result, 1, None, 2)         result = list(izip(evens, odds))          # If you have to have the converter allocate memory,         # deallocate it here...          return result  convert = _BNG_FFI()  if _test_standalone:     print(convert([1.0, 2.0, 3.0], [4.0, 5.0, 6.0], c_float))     print(convert([1, 2, 3], [4, 5, 6], c_uint32))     print(convert([1, 2, 3], (c_uint32 * 3)(4, 5, 6), c_uint32))