Menu
I'd like to send a list of strings to a C function:
from ctypes import c_double, c_void_p, Structure, cast, c_char_p, c_size_t, POINTER
import numpy as np
class FFIArray(Structure):
"""
Convert sequence of structs or types to C-compatible void array
"""
_fields_ = [("data", c_void_p), ("len", c_size_t)]
@classmethod
def from_param(cls, seq):
""" Allow implicit conversions """
return seq if isinstance(seq, cls) else cls(seq)
def __init__(self, seq, data_type):
array = np.ctypeslib.as_array((data_type * len(seq))(*seq))
self._buffer = array.data
self.data = cast(array.ctypes.data_as(POINTER(data_type)), c_void_p)
self.len = len(array)
class Coordinates(Structure):
_fields_ = [("lat", c_double), ("lon", c_double)]
def __str__(self):
return "Latitude: {}, Longitude: {}".format(self.lat, self.lon)
if __name__ == "__main__":
tup = Coordinates(0.0, 1.0)
coords = [tup, tup]
a = b"foo"
b = b"bar"
words = [a, b]
coord_array = FFIArray(coords, data_type=Coordinates)
print(coord_array)
word_array = FFIArray(words, c_char_p)
print(word_array)
This works for e.g. c_double, but fails when I try it with c_char_p, with the following error (testing on Python 2.7.16 and 3.7.4, and NumPy 1.16.5, 1.17.2):
Traceback (most recent call last):
File "/Users/sth/dev/test/venv3/lib/python3.7/site-packages/numpy/core/_internal.py", line 600, in _dtype_from_pep3118
dtype, align = __dtype_from_pep3118(stream, is_subdtype=False)
File "/Users/sth/dev/test/venv3/lib/python3.7/site-packages/numpy/core/_internal.py", line 677, in __dtype_from_pep3118
raise ValueError("Unknown PEP 3118 data type specifier %r" % stream.s)
ValueError: Unknown PEP 3118 data type specifier 'z'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "so_example.py", line 42, in <module>
word_array = FFIArray(words, c_char_p)
File "so_example.py", line 19, in __init__
array = np.ctypeslib.as_array((data_type * len(seq))(*seq))
File "/Users/sth/dev/test/venv3/lib/python3.7/site-packages/numpy/ctypeslib.py", line 523, in as_array
return array(obj, copy=False)
ValueError: '<z' is not a valid PEP 3118 buffer format string
Is there a better way to do this? I'm also not wedded to using numpy, although it's useful for converting iterables of numeric types and numpy arrays to _FFIArray elsewhere.
450
Listing [Python.Docs]: ctypes - A foreign function library for Python.
I didn't (yet) get to the bottom of NumPy's error (so far I reached _multiarray_umath (C) sources, but I don't know how the functions from _internal.py are being called).
In the meantime, here's a variant that doesn't use NumPy (which is not needed in this case, but you mentioned you use it in other parts, so this probably fixes only part of your problem).
code03.py:
#!/usr/bin/env python3
import sys
import ctypes
import numpy as np
class FFIArray(ctypes.Structure):
"""
Convert sequence of structs or types to C-compatible void array
"""
_fields_ = [
("data", ctypes.c_void_p),
("len", ctypes.c_size_t)
]
@classmethod
def from_param(cls, seq, data_type):
""" Allow implicit conversions """
return seq if isinstance(seq, cls) else cls(seq, data_type)
def __init__(self, seq, data_type):
self.len = len(seq)
self._data_type = data_type
self._DataTypeArr = self._data_type * self.len
self.data = ctypes.cast(self._DataTypeArr(*seq), ctypes.c_void_p)
def __str__(self):
ret = super().__str__() # Python 3
#ret = super(FFIArray, self).__str__() # !!! Python 2 !!!
ret += "\nType: {0:s}\nLength: {1:d}\nElement Type: {2:}\nElements:\n".format(
self.__class__.__name__, self.len, self._data_type)
arr_data = self._DataTypeArr.from_address(self.data)
for idx, item in enumerate(arr_data):
ret += " {0:d}: {1:}\n".format(idx, item)
return ret
class Coordinates(ctypes.Structure):
_fields_ = [
("lat", ctypes.c_double),
("lon", ctypes.c_double)
]
def __str__(self):
return "Latitude: {0:.3f}, Longitude: {1:.3f}".format(self.lat, self.lon)
def main():
coord_list = [Coordinates(i+ 1, i * 2) for i in range(4)]
s0 = b"foo"
s1 = b"bar"
word_list = [s0, s1]
coord_array = FFIArray(coord_list, data_type=Coordinates)
print(coord_array)
word_array = FFIArray(word_list, ctypes.c_char_p)
print(word_array)
if __name__ == "__main__":
print("Python {0:s} {1:d}bit on {2:s}\n".format(" ".join(item.strip() for item in sys.version.split("\n")), 64 if sys.maxsize > 0x100000000 else 32, sys.platform))
print("NumPy: {0:s}\n".format(np.version.version))
main()
print("\nDone.")
Notes:
Output:
[cfati@CFATI-5510-0:e:\Work\Dev\StackOverflow\q058049957]> "e:\Work\Dev\VEnvs\py_064_03.07.03_test0\Scripts\python.exe" code03.py Python 3.7.3 (v3.7.3:ef4ec6ed12, Mar 25 2019, 22:22:05) [MSC v.1916 64 bit (AMD64)] 64bit on win32 NumPy: 1.16.2 <__main__.FFIArray object at 0x0000019CFEB63648> Type: FFIArray Length: 4 Element Type: <class '__main__.Coordinates'> Elements: 0: Latitude: 1.000, Longitude: 0.000 1: Latitude: 2.000, Longitude: 2.000 2: Latitude: 3.000, Longitude: 4.000 3: Latitude: 4.000, Longitude: 6.000 <__main__.FFIArray object at 0x0000019CFEB637C8> Type: FFIArray Length: 2 Element Type: <class 'ctypes.c_char_p'> Elements: 0: b'foo' 1: b'bar' Done.
PEP 3118 defines a standard for accessing (sharing) memory. Part of that are the format string specifiers used to do the conversions between a buffer contents and relevant data. Those are listed in [Python.Docs]: PEP 3118 - Additions to the struct string-syntax and extend the ones from [Python 3.Docs]: struct - Format Characters.
ctypes types have an (!!!undocumented!!!) _type_ attribute which (I presume) is used when performing the conversion from / to np:
>>> import ctypes >>> >>> data_types = list() >>> >>> for attr_name in dir(ctypes): ... attr = getattr(ctypes, attr_name, None) ... if isinstance(attr, (type,)) and issubclass(attr, (ctypes._SimpleCData,)): ... data_types.append((attr, attr_name)) ... >>> for data_type, data_type_name in data_types: ... print("{0:} ({1:}) - {2:}".format(data_type, data_type_name, getattr(data_type, "_type_", None))) ... <class 'ctypes.HRESULT'> (HRESULT) - l <class '_ctypes._SimpleCData'> (_SimpleCData) - None <class 'ctypes.c_bool'> (c_bool) - ? <class 'ctypes.c_byte'> (c_byte) - b <class 'ctypes.c_char'> (c_char) - c <class 'ctypes.c_char_p'> (c_char_p) - z <class 'ctypes.c_double'> (c_double) - d <class 'ctypes.c_float'> (c_float) - f <class 'ctypes.c_long'> (c_int) - l <class 'ctypes.c_short'> (c_int16) - h <class 'ctypes.c_long'> (c_int32) - l <class 'ctypes.c_longlong'> (c_int64) - q <class 'ctypes.c_byte'> (c_int8) - b <class 'ctypes.c_long'> (c_long) - l <class 'ctypes.c_double'> (c_longdouble) - d <class 'ctypes.c_longlong'> (c_longlong) - q <class 'ctypes.c_short'> (c_short) - h <class 'ctypes.c_ulonglong'> (c_size_t) - Q <class 'ctypes.c_longlong'> (c_ssize_t) - q <class 'ctypes.c_ubyte'> (c_ubyte) - B <class 'ctypes.c_ulong'> (c_uint) - L <class 'ctypes.c_ushort'> (c_uint16) - H <class 'ctypes.c_ulong'> (c_uint32) - L <class 'ctypes.c_ulonglong'> (c_uint64) - Q <class 'ctypes.c_ubyte'> (c_uint8) - B <class 'ctypes.c_ulong'> (c_ulong) - L <class 'ctypes.c_ulonglong'> (c_ulonglong) - Q <class 'ctypes.c_ushort'> (c_ushort) - H <class 'ctypes.c_void_p'> (c_void_p) - P <class 'ctypes.c_void_p'> (c_voidp) - P <class 'ctypes.c_wchar'> (c_wchar) - u <class 'ctypes.c_wchar_p'> (c_wchar_p) - Z <class 'ctypes.py_object'> (py_object) - O
As seen above, c_char_p and c_whar_p's are not found or don't match the standard. At 1st glance, it seems it's a ctypes bug as it doesn't respect the standard, but I wouldn't rush into claiming this fact (and maybe submit a bug) before further investigations (especially because bugs have already been reported in this area: [Python.Bugs]: ctypes arrays have incorrect buffer information (PEP-3118)).
Below is a variant that also handles np arrays.
code04.py:
#!/usr/bin/env python3
import sys
import ctypes
import numpy as np
class FFIArray(ctypes.Structure):
"""
Convert sequence of structs or types to C-compatible void array
"""
_fields_ = [
("data", ctypes.c_void_p),
("len", ctypes.c_size_t)
]
_special_np_types_mapping = {
ctypes.c_char_p: "S",
ctypes.c_wchar_p: "U",
}
@classmethod
def from_param(cls, seq, data_type=ctypes.c_void_p):
""" Allow implicit conversions """
return seq if isinstance(seq, cls) else cls(seq, data_type=data_type)
def __init__(self, seq, data_type=ctypes.c_void_p):
self.len = len(seq)
self.__data_type = data_type # Used just to hold the value passed to the initializer
if isinstance(seq, np.ndarray):
arr = np.ctypeslib.as_ctypes(seq)
self._data_type = arr._type_ # !!! data_type is ignored in this case !!!
self._DataTypeArr = arr.__class__
self.data = ctypes.cast(arr, ctypes.c_void_p)
else:
self._data_type = data_type
self._DataTypeArr = self._data_type * self.len
self.data = ctypes.cast(self._DataTypeArr(*seq), ctypes.c_void_p)
def __str__(self):
strings = [super().__str__()] # Python 3
#strings = [super(FFIArray, self).__str__()] # !!! Python 2 (ugly) !!!
strings.append("Type: {0:s}\nElement Type: {1:}{2:}\nElements ({3:d}):".format(
self.__class__.__name__, self._data_type,
"" if self._data_type == self.__data_type else " ({0:})".format(self.__data_type),
self.len))
arr_data = self._DataTypeArr.from_address(self.data)
for idx, item in enumerate(arr_data):
strings.append(" {0:d}: {1:}".format(idx, item))
return "\n".join(strings) + "\n"
def to_np(self):
arr_data = self._DataTypeArr.from_address(self.data)
if self._data_type in self._special_np_types_mapping:
dtype = np.dtype(self._special_np_types_mapping[self._data_type] + str(max(len(item) for item in arr_data)))
np_arr = np.empty(self.len, dtype=dtype)
for idx, item in enumerate(arr_data):
np_arr[idx] = item
return np_arr
else:
return np.ctypeslib.as_array(arr_data)
class Coordinates(ctypes.Structure):
_fields_ = [
("lat", ctypes.c_double),
("lon", ctypes.c_double)
]
def __str__(self):
return "Latitude: {0:.3f}, Longitude: {1:.3f}".format(self.lat, self.lon)
def main():
coord_list = [Coordinates(i + 1, i * 2) for i in range(4)]
s0 = b"foo"
s1 = b"bar (beyond all recognition)" # To avoid having 2 equal strings
word_list = [s0, s1]
coord_array0 = FFIArray(coord_list, data_type=Coordinates)
print(coord_array0)
word_array0 = FFIArray(word_list, data_type=ctypes.c_char_p)
print(word_array0)
print("to_np: {0:}\n".format(word_array0.to_np()))
np_array_src = np.array([0, -3.141593, 2.718282, -0.577, 0.618])
float_array0 = FFIArray.from_param(np_array_src, data_type=None)
print(float_array0)
np_array_dst = float_array0.to_np()
print("to_np: {0:}".format(np_array_dst))
print("Equal np arrays: {0:}\n".format(all(np_array_src == np_array_dst)))
empty_array0 = FFIArray.from_param([])
print(empty_array0)
if __name__ == "__main__":
print("Python {0:s} {1:d}bit on {2:s}\n".format(" ".join(item.strip() for item in sys.version.split("\n")), 64 if sys.maxsize > 0x100000000 else 32, sys.platform))
print("NumPy: {0:s}\n".format(np.version.version))
main()
print("\nDone.")
Output:
[cfati@CFATI-5510-0:e:\Work\Dev\StackOverflow\q058049957]> "e:\Work\Dev\VEnvs\py_064_03.07.03_test0\Scripts\python.exe" code04.py Python 3.7.3 (v3.7.3:ef4ec6ed12, Mar 25 2019, 22:22:05) [MSC v.1916 64 bit (AMD64)] 64bit on win32 NumPy: 1.16.2 <__main__.FFIArray object at 0x000002484A2265C8> Type: FFIArray Element Type: <class '__main__.Coordinates'> Elements (4): 0: Latitude: 1.000, Longitude: 0.000 1: Latitude: 2.000, Longitude: 2.000 2: Latitude: 3.000, Longitude: 4.000 3: Latitude: 4.000, Longitude: 6.000 <__main__.FFIArray object at 0x000002484A2267C8> Type: FFIArray Element Type: <class 'ctypes.c_char_p'> Elements (2): 0: b'foo' 1: b'bar (beyond all recognition)' to_np: [b'foo' b'bar (beyond all recognition)'] <__main__.FFIArray object at 0x000002484A2264C8> Type: FFIArray Element Type: <class 'ctypes.c_double'> (None) Elements (5): 0: 0.0 1: -3.141593 2: 2.718282 3: -0.577 4: 0.618 to_np: [ 0. -3.141593 2.718282 -0.577 0.618 ] Equal np arrays: True <__main__.FFIArray object at 0x000002484A226848> Type: FFIArray Element Type: <class 'ctypes.c_void_p'> Elements (0): Done.
Of course, this is one of the possibilities. Another one could involve (deprecated) [SciPy.Docs]: numpy.char.array usage, but I didn't want to overcomplicate things (without a clear scenario).
Added FFIArray to np array conversion (I'm not a np expert, so it might look cumbersome for one who is). Strings need special processing.
Didn't post a new code version (as the changes are not very significant), worked on previous one instead.
113
If you love us? You can donate to us via Paypal or buy me a coffee so we can maintain and grow! Thank you!
Donate Us With
