diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index d67261c461f..04ec0270148 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -14,6 +14,7 @@ from _ctypes import RTLD_LOCAL, RTLD_GLOBAL from _ctypes import ArgumentError from _ctypes import SIZEOF_TIME_T +from _ctypes import CField from struct import calcsize as _calcsize @@ -21,7 +22,7 @@ raise Exception("Version number mismatch", __version__, _ctypes_version) if _os.name == "nt": - from _ctypes import FormatError + from _ctypes import COMError, CopyComPointer, FormatError DEFAULT_MODE = RTLD_LOCAL if _os.name == "posix" and _sys.platform == "darwin": @@ -208,6 +209,18 @@ class c_longdouble(_SimpleCData): if sizeof(c_longdouble) == sizeof(c_double): c_longdouble = c_double +try: + class c_double_complex(_SimpleCData): + _type_ = "D" + _check_size(c_double_complex) + class c_float_complex(_SimpleCData): + _type_ = "F" + _check_size(c_float_complex) + class c_longdouble_complex(_SimpleCData): + _type_ = "G" +except AttributeError: + pass + if _calcsize("l") == _calcsize("q"): # if long and long long have the same size, make c_longlong an alias for c_long c_longlong = c_long @@ -255,7 +268,72 @@ class c_void_p(_SimpleCData): class c_bool(_SimpleCData): _type_ = "?" -from _ctypes import POINTER, pointer, _pointer_type_cache +def POINTER(cls): + """Create and return a new ctypes pointer type. + + Pointer types are cached and reused internally, + so calling this function repeatedly is cheap. + """ + if cls is None: + return c_void_p + try: + return cls.__pointer_type__ + except AttributeError: + pass + if isinstance(cls, str): + # handle old-style incomplete types (see test_ctypes.test_incomplete) + import warnings + warnings._deprecated("ctypes.POINTER with string", remove=(3, 19)) + try: + return _pointer_type_cache_fallback[cls] + except KeyError: + result = type(f'LP_{cls}', (_Pointer,), {}) + _pointer_type_cache_fallback[cls] = result + return result + + # create pointer type and set __pointer_type__ for cls + return type(f'LP_{cls.__name__}', (_Pointer,), {'_type_': cls}) + +def pointer(obj): + """Create a new pointer instance, pointing to 'obj'. + + The returned object is of the type POINTER(type(obj)). Note that if you + just want to pass a pointer to an object to a foreign function call, you + should use byref(obj) which is much faster. + """ + typ = POINTER(type(obj)) + return typ(obj) + +class _PointerTypeCache: + def __setitem__(self, cls, pointer_type): + import warnings + warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19)) + try: + cls.__pointer_type__ = pointer_type + except AttributeError: + _pointer_type_cache_fallback[cls] = pointer_type + + def __getitem__(self, cls): + import warnings + warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19)) + try: + return cls.__pointer_type__ + except AttributeError: + return _pointer_type_cache_fallback[cls] + + def get(self, cls, default=None): + import warnings + warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19)) + try: + return cls.__pointer_type__ + except AttributeError: + return _pointer_type_cache_fallback.get(cls, default) + + def __contains__(self, cls): + return hasattr(cls, '__pointer_type__') + +_pointer_type_cache_fallback = {} +_pointer_type_cache = _PointerTypeCache() class c_wchar_p(_SimpleCData): _type_ = "Z" @@ -266,7 +344,7 @@ class c_wchar(_SimpleCData): _type_ = "u" def _reset_cache(): - _pointer_type_cache.clear() + _pointer_type_cache_fallback.clear() _c_functype_cache.clear() if _os.name == "nt": _win_functype_cache.clear() @@ -274,7 +352,6 @@ def _reset_cache(): POINTER(c_wchar).from_param = c_wchar_p.from_param # _SimpleCData.c_char_p_from_param POINTER(c_char).from_param = c_char_p.from_param - _pointer_type_cache[None] = c_void_p def create_unicode_buffer(init, size=None): """create_unicode_buffer(aString) -> character array @@ -308,13 +385,7 @@ def create_unicode_buffer(init, size=None): def SetPointerType(pointer, cls): import warnings warnings._deprecated("ctypes.SetPointerType", remove=(3, 15)) - if _pointer_type_cache.get(cls, None) is not None: - raise RuntimeError("This type already exists in the cache") - if id(pointer) not in _pointer_type_cache: - raise RuntimeError("What's this???") pointer.set_type(cls) - _pointer_type_cache[cls] = pointer - del _pointer_type_cache[id(pointer)] def ARRAY(typ, len): return typ * len @@ -521,6 +592,7 @@ def WinError(code=None, descr=None): # functions from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr +from _ctypes import _memoryview_at_addr ## void *memmove(void *, const void *, size_t); memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr) @@ -546,6 +618,14 @@ def string_at(ptr, size=-1): Return the byte string at void *ptr.""" return _string_at(ptr, size) +_memoryview_at = PYFUNCTYPE( + py_object, c_void_p, c_ssize_t, c_int)(_memoryview_at_addr) +def memoryview_at(ptr, size, readonly=False): + """memoryview_at(ptr, size[, readonly]) -> memoryview + + Return a memoryview representing the memory at void *ptr.""" + return _memoryview_at(ptr, size, bool(readonly)) + try: from _ctypes import _wstring_at_addr except ImportError: diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py new file mode 100644 index 00000000000..2048ccb6a1c --- /dev/null +++ b/Lib/ctypes/_layout.py @@ -0,0 +1,330 @@ +"""Python implementation of computing the layout of a struct/union + +This code is internal and tightly coupled to the C part. The interface +may change at any time. +""" + +import sys +import warnings + +from _ctypes import CField, buffer_info +import ctypes + +def round_down(n, multiple): + assert n >= 0 + assert multiple > 0 + return (n // multiple) * multiple + +def round_up(n, multiple): + assert n >= 0 + assert multiple > 0 + return ((n + multiple - 1) // multiple) * multiple + +_INT_MAX = (1 << (ctypes.sizeof(ctypes.c_int) * 8) - 1) - 1 + + +class StructUnionLayout: + def __init__(self, fields, size, align, format_spec): + # sequence of CField objects + self.fields = fields + + # total size of the aggregate (rounded up to alignment) + self.size = size + + # total alignment requirement of the aggregate + self.align = align + + # buffer format specification (as a string, UTF-8 but bes + # kept ASCII-only) + self.format_spec = format_spec + + +def get_layout(cls, input_fields, is_struct, base): + """Return a StructUnionLayout for the given class. + + Called by PyCStructUnionType_update_stginfo when _fields_ is assigned + to a class. + """ + # Currently there are two modes, selectable using the '_layout_' attribute: + # + # 'gcc-sysv' mode places fields one after another, bit by bit. + # But "each bit field must fit within a single object of its specified + # type" (GCC manual, section 15.8 "Bit Field Packing"). When it doesn't, + # we insert a few bits of padding to avoid that. + # + # 'ms' mode works similar except for bitfield packing. Adjacent + # bit-fields are packed into the same 1-, 2-, or 4-byte allocation unit + # if the integral types are the same size and if the next bit-field fits + # into the current allocation unit without crossing the boundary imposed + # by the common alignment requirements of the bit-fields. + # + # See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-mms-bitfields + # for details. + + # We do not support zero length bitfields (we use bitsize != 0 + # elsewhere to indicate a bitfield). Here, non-bitfields have bit_size + # set to size*8. + + # For clarity, variables that count bits have `bit` in their names. + + pack = getattr(cls, '_pack_', None) + + layout = getattr(cls, '_layout_', None) + if layout is None: + if sys.platform == 'win32': + gcc_layout = False + elif pack: + if is_struct: + base_type_name = 'Structure' + else: + base_type_name = 'Union' + warnings._deprecated( + '_pack_ without _layout_', + f"Due to '_pack_', the '{cls.__name__}' {base_type_name} will " + + "use memory layout compatible with MSVC (Windows). " + + "If this is intended, set _layout_ to 'ms'. " + + "The implicit default is deprecated and slated to become " + + "an error in Python {remove}.", + remove=(3, 19), + ) + gcc_layout = False + else: + gcc_layout = True + elif layout == 'ms': + gcc_layout = False + elif layout == 'gcc-sysv': + gcc_layout = True + else: + raise ValueError(f'unknown _layout_: {layout!r}') + + align = getattr(cls, '_align_', 1) + if align < 0: + raise ValueError('_align_ must be a non-negative integer') + elif align == 0: + # Setting `_align_ = 0` amounts to using the default alignment + align = 1 + + if base: + align = max(ctypes.alignment(base), align) + + swapped_bytes = hasattr(cls, '_swappedbytes_') + if swapped_bytes: + big_endian = sys.byteorder == 'little' + else: + big_endian = sys.byteorder == 'big' + + if pack is not None: + try: + pack = int(pack) + except (TypeError, ValueError): + raise ValueError("_pack_ must be an integer") + if pack < 0: + raise ValueError("_pack_ must be a non-negative integer") + if pack > _INT_MAX: + raise ValueError("_pack_ too big") + if gcc_layout: + raise ValueError('_pack_ is not compatible with gcc-sysv layout') + + result_fields = [] + + if is_struct: + format_spec_parts = ["T{"] + else: + format_spec_parts = ["B"] + + last_field_bit_size = 0 # used in MS layout only + + # `8 * next_byte_offset + next_bit_offset` points to where the + # next field would start. + next_bit_offset = 0 + next_byte_offset = 0 + + # size if this was a struct (sum of field sizes, plus padding) + struct_size = 0 + # max of field sizes; only meaningful for unions + union_size = 0 + + if base: + struct_size = ctypes.sizeof(base) + if gcc_layout: + next_bit_offset = struct_size * 8 + else: + next_byte_offset = struct_size + + last_size = struct_size + for i, field in enumerate(input_fields): + if not is_struct: + # Unions start fresh each time + last_field_bit_size = 0 + next_bit_offset = 0 + next_byte_offset = 0 + + # Unpack the field + field = tuple(field) + try: + name, ctype = field + except (ValueError, TypeError): + try: + name, ctype, bit_size = field + except (ValueError, TypeError) as exc: + raise ValueError( + '_fields_ must be a sequence of (name, C type) pairs ' + + 'or (name, C type, bit size) triples') from exc + is_bitfield = True + if bit_size <= 0: + raise ValueError( + f'number of bits invalid for bit field {name!r}') + type_size = ctypes.sizeof(ctype) + if bit_size > type_size * 8: + raise ValueError( + f'number of bits invalid for bit field {name!r}') + else: + is_bitfield = False + type_size = ctypes.sizeof(ctype) + bit_size = type_size * 8 + + type_bit_size = type_size * 8 + type_align = ctypes.alignment(ctype) or 1 + type_bit_align = type_align * 8 + + if gcc_layout: + # We don't use next_byte_offset here + assert pack is None + assert next_byte_offset == 0 + + # Determine whether the bit field, if placed at the next + # free bit, fits within a single object of its specified type. + # That is: determine a "slot", sized & aligned for the + # specified type, which contains the bitfield's beginning: + slot_start_bit = round_down(next_bit_offset, type_bit_align) + slot_end_bit = slot_start_bit + type_bit_size + # And see if it also contains the bitfield's last bit: + field_end_bit = next_bit_offset + bit_size + if field_end_bit > slot_end_bit: + # It doesn't: add padding (bump up to the next + # alignment boundary) + next_bit_offset = round_up(next_bit_offset, type_bit_align) + + offset = round_down(next_bit_offset, type_bit_align) // 8 + if is_bitfield: + bit_offset = next_bit_offset - 8 * offset + assert bit_offset <= type_bit_size + else: + assert offset == next_bit_offset / 8 + + next_bit_offset += bit_size + struct_size = round_up(next_bit_offset, 8) // 8 + else: + if pack: + type_align = min(pack, type_align) + + # next_byte_offset points to end of current bitfield. + # next_bit_offset is generally non-positive, + # and 8 * next_byte_offset + next_bit_offset points just behind + # the end of the last field we placed. + if ( + (0 < next_bit_offset + bit_size) + or (type_bit_size != last_field_bit_size) + ): + # Close the previous bitfield (if any) + # and start a new bitfield + next_byte_offset = round_up(next_byte_offset, type_align) + + next_byte_offset += type_size + + last_field_bit_size = type_bit_size + # Reminder: 8 * (next_byte_offset) + next_bit_offset + # points to where we would start a new field, namely + # just behind where we placed the last field plus an + # allowance for alignment. + next_bit_offset = -last_field_bit_size + + assert type_bit_size == last_field_bit_size + + offset = next_byte_offset - last_field_bit_size // 8 + if is_bitfield: + assert 0 <= (last_field_bit_size + next_bit_offset) + bit_offset = last_field_bit_size + next_bit_offset + if type_bit_size: + assert (last_field_bit_size + next_bit_offset) < type_bit_size + + next_bit_offset += bit_size + struct_size = next_byte_offset + + if is_bitfield and big_endian: + # On big-endian architectures, bit fields are also laid out + # starting with the big end. + bit_offset = type_bit_size - bit_size - bit_offset + + # Add the format spec parts + if is_struct: + padding = offset - last_size + format_spec_parts.append(padding_spec(padding)) + + fieldfmt, bf_ndim, bf_shape = buffer_info(ctype) + + if bf_shape: + format_spec_parts.extend(( + "(", + ','.join(str(n) for n in bf_shape), + ")", + )) + + if fieldfmt is None: + fieldfmt = "B" + if isinstance(name, bytes): + # a bytes name would be rejected later, but we check early + # to avoid a BytesWarning with `python -bb` + raise TypeError( + f"field {name!r}: name must be a string, not bytes") + format_spec_parts.append(f"{fieldfmt}:{name}:") + + result_fields.append(CField( + name=name, + type=ctype, + byte_size=type_size, + byte_offset=offset, + bit_size=bit_size if is_bitfield else None, + bit_offset=bit_offset if is_bitfield else None, + index=i, + + # Do not use CField outside ctypes, yet. + # The constructor is internal API and may change without warning. + _internal_use=True, + )) + if is_bitfield and not gcc_layout: + assert type_bit_size > 0 + + align = max(align, type_align) + last_size = struct_size + if not is_struct: + union_size = max(struct_size, union_size) + + if is_struct: + total_size = struct_size + else: + total_size = union_size + + # Adjust the size according to the alignment requirements + aligned_size = round_up(total_size, align) + + # Finish up the format spec + if is_struct: + padding = aligned_size - total_size + format_spec_parts.append(padding_spec(padding)) + format_spec_parts.append("}") + + return StructUnionLayout( + fields=result_fields, + size=aligned_size, + align=align, + format_spec="".join(format_spec_parts), + ) + + +def padding_spec(padding): + if padding <= 0: + return "" + if padding == 1: + return "x" + return f"{padding}x" diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 117bf06cb01..378f12167c6 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -67,6 +67,65 @@ def find_library(name): return fname return None + # Listing loaded DLLs on Windows relies on the following APIs: + # https://learn.microsoft.com/windows/win32/api/psapi/nf-psapi-enumprocessmodules + # https://learn.microsoft.com/windows/win32/api/libloaderapi/nf-libloaderapi-getmodulefilenamew + import ctypes + from ctypes import wintypes + + _kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + _get_current_process = _kernel32["GetCurrentProcess"] + _get_current_process.restype = wintypes.HANDLE + + _k32_get_module_file_name = _kernel32["GetModuleFileNameW"] + _k32_get_module_file_name.restype = wintypes.DWORD + _k32_get_module_file_name.argtypes = ( + wintypes.HMODULE, + wintypes.LPWSTR, + wintypes.DWORD, + ) + + _psapi = ctypes.WinDLL('psapi', use_last_error=True) + _enum_process_modules = _psapi["EnumProcessModules"] + _enum_process_modules.restype = wintypes.BOOL + _enum_process_modules.argtypes = ( + wintypes.HANDLE, + ctypes.POINTER(wintypes.HMODULE), + wintypes.DWORD, + wintypes.LPDWORD, + ) + + def _get_module_filename(module: wintypes.HMODULE): + name = (wintypes.WCHAR * 32767)() # UNICODE_STRING_MAX_CHARS + if _k32_get_module_file_name(module, name, len(name)): + return name.value + return None + + + def _get_module_handles(): + process = _get_current_process() + space_needed = wintypes.DWORD() + n = 1024 + while True: + modules = (wintypes.HMODULE * n)() + if not _enum_process_modules(process, + modules, + ctypes.sizeof(modules), + ctypes.byref(space_needed)): + err = ctypes.get_last_error() + msg = ctypes.FormatError(err).strip() + raise ctypes.WinError(err, f"EnumProcessModules failed: {msg}") + n = space_needed.value // ctypes.sizeof(wintypes.HMODULE) + if n <= len(modules): + return modules[:n] + + def dllist(): + """Return a list of loaded shared libraries in the current process.""" + modules = _get_module_handles() + libraries = [name for h in modules + if (name := _get_module_filename(h)) is not None] + return libraries + elif os.name == "posix" and sys.platform in {"darwin", "ios", "tvos", "watchos"}: from ctypes.macholib.dyld import dyld_find as _dyld_find def find_library(name): @@ -80,6 +139,22 @@ def find_library(name): continue return None + # Listing loaded libraries on Apple systems relies on the following API: + # https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dyld.3.html + import ctypes + + _libc = ctypes.CDLL(find_library("c")) + _dyld_get_image_name = _libc["_dyld_get_image_name"] + _dyld_get_image_name.restype = ctypes.c_char_p + + def dllist(): + """Return a list of loaded shared libraries in the current process.""" + num_images = _libc._dyld_image_count() + libraries = [os.fsdecode(name) for i in range(num_images) + if (name := _dyld_get_image_name(i)) is not None] + + return libraries + elif sys.platform.startswith("aix"): # AIX has two styles of storing shared libraries # GNU auto_tools refer to these as svr4 and aix @@ -98,6 +173,25 @@ def find_library(name): fname = f"{directory}/lib{name}.so" return fname if os.path.isfile(fname) else None +elif sys.platform == "emscripten": + def _is_wasm(filename): + # Return True if the given file is an WASM module + wasm_header = b"\x00asm" + with open(filename, 'br') as thefile: + return thefile.read(4) == wasm_header + + def find_library(name): + candidates = [f"lib{name}.so", f"lib{name}.wasm"] + paths = os.environ.get("LD_LIBRARY_PATH", "") + for libdir in paths.split(":"): + for name in candidates: + libfile = os.path.join(libdir, name) + + if os.path.isfile(libfile) and _is_wasm(libfile): + return libfile + + return None + elif os.name == "posix": # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump import re, tempfile @@ -341,6 +435,55 @@ def find_library(name): return _findSoname_ldconfig(name) or \ _get_soname(_findLib_gcc(name)) or _get_soname(_findLib_ld(name)) + +# Listing loaded libraries on other systems will try to use +# functions common to Linux and a few other Unix-like systems. +# See the following for several platforms' documentation of the same API: +# https://man7.org/linux/man-pages/man3/dl_iterate_phdr.3.html +# https://man.freebsd.org/cgi/man.cgi?query=dl_iterate_phdr +# https://man.openbsd.org/dl_iterate_phdr +# https://docs.oracle.com/cd/E88353_01/html/E37843/dl-iterate-phdr-3c.html +if (os.name == "posix" and + sys.platform not in {"darwin", "ios", "tvos", "watchos"}): + import ctypes + if hasattr((_libc := ctypes.CDLL(None)), "dl_iterate_phdr"): + + class _dl_phdr_info(ctypes.Structure): + _fields_ = [ + ("dlpi_addr", ctypes.c_void_p), + ("dlpi_name", ctypes.c_char_p), + ("dlpi_phdr", ctypes.c_void_p), + ("dlpi_phnum", ctypes.c_ushort), + ] + + _dl_phdr_callback = ctypes.CFUNCTYPE( + ctypes.c_int, + ctypes.POINTER(_dl_phdr_info), + ctypes.c_size_t, + ctypes.POINTER(ctypes.py_object), + ) + + @_dl_phdr_callback + def _info_callback(info, _size, data): + libraries = data.contents.value + name = os.fsdecode(info.contents.dlpi_name) + libraries.append(name) + return 0 + + _dl_iterate_phdr = _libc["dl_iterate_phdr"] + _dl_iterate_phdr.argtypes = [ + _dl_phdr_callback, + ctypes.POINTER(ctypes.py_object), + ] + _dl_iterate_phdr.restype = ctypes.c_int + + def dllist(): + """Return a list of loaded shared libraries in the current process.""" + libraries = [] + _dl_iterate_phdr(_info_callback, + ctypes.byref(ctypes.py_object(libraries))) + return libraries + ################################################################ # test code @@ -384,5 +527,12 @@ def test(): print(cdll.LoadLibrary("libcrypt.so")) print(find_library("crypt")) + try: + dllist + except NameError: + print('dllist() not available') + else: + print(dllist()) + if __name__ == "__main__": test() diff --git a/Lib/ctypes/wintypes.py b/Lib/ctypes/wintypes.py index 9c4e721438a..4beba0d1951 100644 --- a/Lib/ctypes/wintypes.py +++ b/Lib/ctypes/wintypes.py @@ -63,10 +63,16 @@ def __repr__(self): HBITMAP = HANDLE HBRUSH = HANDLE HCOLORSPACE = HANDLE +HCONV = HANDLE +HCONVLIST = HANDLE +HCURSOR = HANDLE HDC = HANDLE +HDDEDATA = HANDLE HDESK = HANDLE +HDROP = HANDLE HDWP = HANDLE HENHMETAFILE = HANDLE +HFILE = INT HFONT = HANDLE HGDIOBJ = HANDLE HGLOBAL = HANDLE @@ -82,9 +88,11 @@ def __repr__(self): HMONITOR = HANDLE HPALETTE = HANDLE HPEN = HANDLE +HRESULT = LONG HRGN = HANDLE HRSRC = HANDLE HSTR = HANDLE +HSZ = HANDLE HTASK = HANDLE HWINSTA = HANDLE HWND = HANDLE diff --git a/Lib/test/test_ctypes/_support.py b/Lib/test/test_ctypes/_support.py index e4c2b33825a..946d654a19a 100644 --- a/Lib/test/test_ctypes/_support.py +++ b/Lib/test/test_ctypes/_support.py @@ -2,15 +2,13 @@ import ctypes from _ctypes import Structure, Union, _Pointer, Array, _SimpleCData, CFuncPtr +import sys +from test import support _CData = Structure.__base__ assert _CData.__name__ == "_CData" -class _X(Structure): - _fields_ = [("x", ctypes.c_int)] -CField = type(_X.x) - # metaclasses PyCStructType = type(Structure) UnionType = type(Union) @@ -22,3 +20,132 @@ class _X(Structure): # type flags Py_TPFLAGS_DISALLOW_INSTANTIATION = 1 << 7 Py_TPFLAGS_IMMUTABLETYPE = 1 << 8 + + +def is_underaligned(ctype): + """Return true when type's alignment is less than its size. + + A famous example is 64-bit int on 32-bit x86. + """ + return ctypes.alignment(ctype) < ctypes.sizeof(ctype) + + +class StructCheckMixin: + def check_struct(self, structure): + """Assert that a structure is well-formed""" + self._check_struct_or_union(structure, is_struct=True) + + def check_union(self, union): + """Assert that a union is well-formed""" + self._check_struct_or_union(union, is_struct=False) + + def check_struct_or_union(self, cls): + if issubclass(cls, Structure): + self._check_struct_or_union(cls, is_struct=True) + elif issubclass(cls, Union): + self._check_struct_or_union(cls, is_struct=False) + else: + raise TypeError(cls) + + def _check_struct_or_union(self, cls, is_struct): + + # Check that fields are not overlapping (for structs), + # and that their metadata is consistent. + + used_bits = 0 + + is_little_endian = ( + hasattr(cls, '_swappedbytes_') ^ (sys.byteorder == 'little')) + + anon_names = getattr(cls, '_anonymous_', ()) + cls_size = ctypes.sizeof(cls) + for name, requested_type, *rest_of_tuple in cls._fields_: + field = getattr(cls, name) + with self.subTest(name=name, field=field): + is_bitfield = len(rest_of_tuple) > 0 + + # name + self.assertEqual(field.name, name) + + # type + self.assertEqual(field.type, requested_type) + + # offset === byte_offset + self.assertEqual(field.byte_offset, field.offset) + if not is_struct: + self.assertEqual(field.byte_offset, 0) + + # byte_size + self.assertEqual(field.byte_size, ctypes.sizeof(field.type)) + self.assertGreaterEqual(field.byte_size, 0) + + # Check that the field is inside the struct. + # See gh-130410 for why this is skipped for bitfields of + # underaligned types. Later in this function (see `bit_end`) + # we assert that the value *bits* are inside the struct. + if not (field.is_bitfield and is_underaligned(field.type)): + self.assertLessEqual(field.byte_offset + field.byte_size, + cls_size) + + # size + self.assertGreaterEqual(field.size, 0) + if is_bitfield: + # size has backwards-compatible bit-packed info + expected_size = (field.bit_size << 16) + field.bit_offset + self.assertEqual(field.size, expected_size) + else: + # size == byte_size + self.assertEqual(field.size, field.byte_size) + + # is_bitfield (bool) + self.assertIs(field.is_bitfield, is_bitfield) + + # bit_offset + if is_bitfield: + self.assertGreaterEqual(field.bit_offset, 0) + self.assertLessEqual(field.bit_offset + field.bit_size, + field.byte_size * 8) + else: + self.assertEqual(field.bit_offset, 0) + if not is_struct: + if is_little_endian: + self.assertEqual(field.bit_offset, 0) + else: + self.assertEqual(field.bit_offset, + field.byte_size * 8 - field.bit_size) + + # bit_size + if is_bitfield: + self.assertGreaterEqual(field.bit_size, 0) + self.assertLessEqual(field.bit_size, field.byte_size * 8) + [requested_bit_size] = rest_of_tuple + self.assertEqual(field.bit_size, requested_bit_size) + else: + self.assertEqual(field.bit_size, field.byte_size * 8) + + # is_anonymous (bool) + self.assertIs(field.is_anonymous, name in anon_names) + + # In a struct, field should not overlap. + # (Test skipped if the structs is enormous.) + if is_struct and cls_size < 10_000: + # Get a mask indicating where the field is within the struct + if is_little_endian: + tp_shift = field.byte_offset * 8 + else: + tp_shift = (cls_size + - field.byte_offset + - field.byte_size) * 8 + mask = (1 << field.bit_size) - 1 + mask <<= (tp_shift + field.bit_offset) + assert mask.bit_count() == field.bit_size + # Check that these bits aren't shared with previous fields + self.assertEqual(used_bits & mask, 0) + # Mark the bits for future checks + used_bits |= mask + + # field is inside cls + bit_end = (field.byte_offset * 8 + + field.bit_offset + + field.bit_size) + self.assertLessEqual(bit_end, cls_size * 8) diff --git a/Lib/test/test_ctypes/test_aligned_structures.py b/Lib/test/test_ctypes/test_aligned_structures.py index 8e8ac429900..50b4d729b9d 100644 --- a/Lib/test/test_ctypes/test_aligned_structures.py +++ b/Lib/test/test_ctypes/test_aligned_structures.py @@ -1,13 +1,13 @@ from ctypes import ( c_char, c_uint32, c_uint16, c_ubyte, c_byte, alignment, sizeof, BigEndianStructure, LittleEndianStructure, - BigEndianUnion, LittleEndianUnion, Structure + BigEndianUnion, LittleEndianUnion, Structure, ) import struct import unittest +from ._support import StructCheckMixin - -class TestAlignedStructures(unittest.TestCase): +class TestAlignedStructures(unittest.TestCase, StructCheckMixin): def test_aligned_string(self): for base, e in ( (LittleEndianStructure, "<"), @@ -19,12 +19,14 @@ class Aligned(base): _fields_ = [ ('value', c_char * 12) ] + self.check_struct(Aligned) class Main(base): _fields_ = [ ('first', c_uint32), ('string', Aligned), ] + self.check_struct(Main) main = Main.from_buffer(data) self.assertEqual(main.first, 7) @@ -46,12 +48,14 @@ class SomeBools(base): ("bool1", c_ubyte), ("bool2", c_ubyte), ] + self.check_struct(SomeBools) class Main(base): _fields_ = [ ("x", c_ubyte), ("y", SomeBools), ("z", c_ubyte), ] + self.check_struct(Main) main = Main.from_buffer(data) self.assertEqual(alignment(SomeBools), 4) @@ -65,6 +69,41 @@ class Main(base): self.assertEqual(Main.z.offset, 8) self.assertEqual(main.z, 7) + def test_negative_align(self): + for base in (Structure, LittleEndianStructure, BigEndianStructure): + with ( + self.subTest(base=base), + self.assertRaisesRegex( + ValueError, + '_align_ must be a non-negative integer', + ) + ): + class MyStructure(base): + _align_ = -1 + _fields_ = [] + + def test_zero_align_no_fields(self): + for base in (Structure, LittleEndianStructure, BigEndianStructure): + with self.subTest(base=base): + class MyStructure(base): + _align_ = 0 + _fields_ = [] + + self.assertEqual(alignment(MyStructure), 1) + self.assertEqual(alignment(MyStructure()), 1) + + def test_zero_align_with_fields(self): + for base in (Structure, LittleEndianStructure, BigEndianStructure): + with self.subTest(base=base): + class MyStructure(base): + _align_ = 0 + _fields_ = [ + ("x", c_ubyte), + ] + + self.assertEqual(alignment(MyStructure), 1) + self.assertEqual(alignment(MyStructure()), 1) + def test_oversized_structure(self): data = bytearray(b"\0" * 8) for base in (LittleEndianStructure, BigEndianStructure): @@ -75,11 +114,13 @@ class SomeBoolsTooBig(base): ("bool2", c_ubyte), ("bool3", c_ubyte), ] + self.check_struct(SomeBoolsTooBig) class Main(base): _fields_ = [ ("y", SomeBoolsTooBig), ("z", c_uint32), ] + self.check_struct(Main) with self.assertRaises(ValueError) as ctx: Main.from_buffer(data) self.assertEqual( @@ -98,18 +139,21 @@ class UnalignedSub(base): _fields_ = [ ("x", c_uint32), ] + self.check_struct(UnalignedSub) class AlignedStruct(UnalignedSub): _align_ = 8 _fields_ = [ ("y", c_uint32), ] + self.check_struct(AlignedStruct) class Main(base): _fields_ = [ ("a", c_uint32), ("b", AlignedStruct) ] + self.check_struct(Main) main = Main.from_buffer(data) self.assertEqual(alignment(main.b), 8) @@ -134,12 +178,14 @@ class AlignedUnion(ubase): ("a", c_uint32), ("b", c_ubyte * 7), ] + self.check_union(AlignedUnion) class Main(sbase): _fields_ = [ ("first", c_uint32), ("union", AlignedUnion), ] + self.check_struct(Main) main = Main.from_buffer(data) self.assertEqual(main.first, 1) @@ -162,18 +208,21 @@ class Sub(sbase): ("x", c_uint32), ("y", c_uint32), ] + self.check_struct(Sub) class MainUnion(ubase): _fields_ = [ ("a", c_uint32), ("b", Sub), ] + self.check_union(MainUnion) class Main(sbase): _fields_ = [ ("first", c_uint32), ("union", MainUnion), ] + self.check_struct(Main) main = Main.from_buffer(data) self.assertEqual(Main.first.size, 4) @@ -198,17 +247,20 @@ class SubUnion(ubase): ("unsigned", c_ubyte), ("signed", c_byte), ] + self.check_union(SubUnion) class MainUnion(SubUnion): _fields_ = [ ("num", c_uint32) ] + self.check_union(SubUnion) class Main(sbase): _fields_ = [ ("first", c_uint16), ("union", MainUnion), ] + self.check_struct(Main) main = Main.from_buffer(data) self.assertEqual(main.union.num, 0xD60102D7) @@ -232,11 +284,13 @@ class SubUnion(ubase): ("unsigned", c_ubyte), ("signed", c_byte), ] + self.check_union(SubUnion) class Main(SubUnion): _fields_ = [ ("num", c_uint32) ] + self.check_struct(Main) main = Main.from_buffer(data) self.assertEqual(alignment(main), 8) @@ -258,14 +312,17 @@ class Inner(sbase): ("x", c_uint16), ("y", c_uint16), ] + self.check_struct(Inner) class Main(sbase): _pack_ = 1 + _layout_ = "ms" _fields_ = [ ("a", c_ubyte), ("b", Inner), ("c", c_ubyte), ] + self.check_struct(Main) main = Main.from_buffer(data) self.assertEqual(sizeof(main), 10) @@ -281,41 +338,6 @@ class Main(sbase): self.assertEqual(main.b.y, 3) self.assertEqual(main.c, 4) - def test_negative_align(self): - for base in (Structure, LittleEndianStructure, BigEndianStructure): - with ( - self.subTest(base=base), - self.assertRaisesRegex( - ValueError, - '_align_ must be a non-negative integer', - ) - ): - class MyStructure(base): - _align_ = -1 - _fields_ = [] - - def test_zero_align_no_fields(self): - for base in (Structure, LittleEndianStructure, BigEndianStructure): - with self.subTest(base=base): - class MyStructure(base): - _align_ = 0 - _fields_ = [] - - self.assertEqual(alignment(MyStructure), 1) - self.assertEqual(alignment(MyStructure()), 1) - - def test_zero_align_with_fields(self): - for base in (Structure, LittleEndianStructure, BigEndianStructure): - with self.subTest(base=base): - class MyStructure(base): - _align_ = 0 - _fields_ = [ - ("x", c_ubyte), - ] - - self.assertEqual(alignment(MyStructure), 1) - self.assertEqual(alignment(MyStructure()), 1) - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_anon.py b/Lib/test/test_ctypes/test_anon.py index b36397b510f..2e16e708635 100644 --- a/Lib/test/test_ctypes/test_anon.py +++ b/Lib/test/test_ctypes/test_anon.py @@ -1,20 +1,23 @@ import unittest import test.support from ctypes import c_int, Union, Structure, sizeof +from ._support import StructCheckMixin -class AnonTest(unittest.TestCase): +class AnonTest(unittest.TestCase, StructCheckMixin): def test_anon(self): class ANON(Union): _fields_ = [("a", c_int), ("b", c_int)] + self.check_union(ANON) class Y(Structure): _fields_ = [("x", c_int), ("_", ANON), ("y", c_int)] _anonymous_ = ["_"] + self.check_struct(Y) self.assertEqual(Y.a.offset, sizeof(c_int)) self.assertEqual(Y.b.offset, sizeof(c_int)) @@ -52,17 +55,20 @@ class Name(Structure): def test_nested(self): class ANON_S(Structure): _fields_ = [("a", c_int)] + self.check_struct(ANON_S) class ANON_U(Union): _fields_ = [("_", ANON_S), ("b", c_int)] _anonymous_ = ["_"] + self.check_union(ANON_U) class Y(Structure): _fields_ = [("x", c_int), ("_", ANON_U), ("y", c_int)] _anonymous_ = ["_"] + self.check_struct(Y) self.assertEqual(Y.x.offset, 0) self.assertEqual(Y.a.offset, sizeof(c_int)) diff --git a/Lib/test/test_ctypes/test_arrays.py b/Lib/test/test_ctypes/test_arrays.py index c80fdff5de6..7f1f6cf5840 100644 --- a/Lib/test/test_ctypes/test_arrays.py +++ b/Lib/test/test_ctypes/test_arrays.py @@ -5,7 +5,7 @@ create_string_buffer, create_unicode_buffer, c_char, c_wchar, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulonglong, c_float, c_double, c_longdouble) -from test.support import bigmemtest, _2G +from test.support import bigmemtest, _2G, threading_helper, Py_GIL_DISABLED from ._support import (_CData, PyCArrayType, Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE) @@ -267,6 +267,26 @@ def test_bpo36504_signed_int_overflow(self): def test_large_array(self, size): c_char * size + @threading_helper.requires_working_threading() + @unittest.skipUnless(Py_GIL_DISABLED, "only meaningful if the GIL is disabled") + def test_thread_safety(self): + from threading import Thread + + buffer = (ctypes.c_char_p * 10)() + + def run(): + for i in range(100): + buffer.value = b"hello" + buffer[0] = b"j" + + with threading_helper.catch_threading_exception() as cm: + threads = (Thread(target=run) for _ in range(25)) + with threading_helper.start_threads(threads): + pass + + if cm.exc_value: + raise cm.exc_value + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_as_parameter.py b/Lib/test/test_ctypes/test_as_parameter.py index c5e1840b0eb..2da1acfcf29 100644 --- a/Lib/test/test_ctypes/test_as_parameter.py +++ b/Lib/test/test_ctypes/test_as_parameter.py @@ -5,7 +5,7 @@ c_short, c_int, c_long, c_longlong, c_byte, c_wchar, c_float, c_double, ArgumentError) -from test.support import import_helper +from test.support import import_helper, skip_if_sanitizer _ctypes_test = import_helper.import_module("_ctypes_test") @@ -192,6 +192,7 @@ class S8I(Structure): self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h), (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) + @skip_if_sanitizer('requires deep stack', thread=True) def test_recursive_as_param(self): class A: pass diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 0332544b582..518f838219e 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -1,72 +1,137 @@ import os +import sys import unittest from ctypes import (CDLL, Structure, sizeof, POINTER, byref, alignment, LittleEndianStructure, BigEndianStructure, c_byte, c_ubyte, c_char, c_char_p, c_void_p, c_wchar, - c_uint32, c_uint64, - c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong) + c_uint8, c_uint16, c_uint32, c_uint64, + c_short, c_ushort, c_int, c_uint, c_long, c_ulong, + c_longlong, c_ulonglong, + Union) from test import support from test.support import import_helper +from ._support import StructCheckMixin _ctypes_test = import_helper.import_module("_ctypes_test") +TEST_FIELDS = ( + ("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9), + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7), +) + + class BITS(Structure): - _fields_ = [("A", c_int, 1), - ("B", c_int, 2), - ("C", c_int, 3), - ("D", c_int, 4), - ("E", c_int, 5), - ("F", c_int, 6), - ("G", c_int, 7), - ("H", c_int, 8), - ("I", c_int, 9), - - ("M", c_short, 1), - ("N", c_short, 2), - ("O", c_short, 3), - ("P", c_short, 4), - ("Q", c_short, 5), - ("R", c_short, 6), - ("S", c_short, 7)] + _fields_ = TEST_FIELDS func = CDLL(_ctypes_test.__file__).unpack_bitfields func.argtypes = POINTER(BITS), c_char +class BITS_msvc(Structure): + _layout_ = "ms" + _fields_ = TEST_FIELDS + + +class BITS_gcc(Structure): + _layout_ = "gcc-sysv" + _fields_ = TEST_FIELDS + + +try: + func_msvc = CDLL(_ctypes_test.__file__).unpack_bitfields_msvc +except AttributeError as err: + # The MSVC struct must be available on Windows; it's optional elsewhere + if support.MS_WINDOWS: + raise err + func_msvc = None +else: + func_msvc.argtypes = POINTER(BITS_msvc), c_char + + class C_Test(unittest.TestCase): def test_ints(self): for i in range(512): for name in "ABCDEFGHI": - b = BITS() - setattr(b, name, i) - self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) + with self.subTest(i=i, name=name): + b = BITS() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func(byref(b), (name.encode('ascii')))) - # bpo-46913: _ctypes/cfield.c h_get() has an undefined behavior - @support.skip_if_sanitizer(ub=True) def test_shorts(self): b = BITS() name = "M" + # See Modules/_ctypes/_ctypes_test.c for where the magic 999 comes from. if func(byref(b), name.encode('ascii')) == 999: + # unpack_bitfields and unpack_bitfields_msvc in + # Modules/_ctypes/_ctypes_test.c return 999 to indicate + # an invalid name. 'M' is only valid, if signed short bitfields + # are supported by the C compiler. self.skipTest("Compiler does not support signed short bitfields") for i in range(256): for name in "MNOPQRS": - b = BITS() - setattr(b, name, i) - self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) + with self.subTest(i=i, name=name): + b = BITS() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func(byref(b), (name.encode('ascii')))) + + @unittest.skipUnless(func_msvc, "need MSVC or __attribute__((ms_struct))") + def test_shorts_msvc_mode(self): + b = BITS_msvc() + name = "M" + # See Modules/_ctypes/_ctypes_test.c for where the magic 999 comes from. + if func_msvc(byref(b), name.encode('ascii')) == 999: + # unpack_bitfields and unpack_bitfields_msvc in + # Modules/_ctypes/_ctypes_test.c return 999 to indicate + # an invalid name. 'M' is only valid, if signed short bitfields + # are supported by the C compiler. + self.skipTest("Compiler does not support signed short bitfields") + for i in range(256): + for name in "MNOPQRS": + with self.subTest(i=i, name=name): + b = BITS_msvc() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func_msvc(byref(b), name.encode('ascii'))) signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong) unsigned_int_types = (c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong) int_types = unsigned_int_types + signed_int_types -class BitFieldTest(unittest.TestCase): +class BitFieldTest(unittest.TestCase, StructCheckMixin): + + def test_generic_checks(self): + self.check_struct(BITS) + self.check_struct(BITS_msvc) + self.check_struct(BITS_gcc) def test_longlong(self): class X(Structure): _fields_ = [("a", c_longlong, 1), ("b", c_longlong, 62), ("c", c_longlong, 1)] + self.check_struct(X) self.assertEqual(sizeof(X), sizeof(c_longlong)) x = X() @@ -78,6 +143,7 @@ class X(Structure): _fields_ = [("a", c_ulonglong, 1), ("b", c_ulonglong, 62), ("c", c_ulonglong, 1)] + self.check_struct(X) self.assertEqual(sizeof(X), sizeof(c_longlong)) x = X() @@ -87,39 +153,49 @@ class X(Structure): def test_signed(self): for c_typ in signed_int_types: - class X(Structure): - _fields_ = [("dummy", c_typ), - ("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)*2) - - x = X() - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) - x.a = -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0)) - x.a, x.b = 0, -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0)) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + class X(Structure): + _fields_ = [("dummy", c_typ), + ("a", c_typ, 3), + ("b", c_typ, 3), + ("c", c_typ, 1)] + self.check_struct(X) + self.assertEqual(sizeof(X), sizeof(c_typ)*2) + + x = X() + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) + x.a = -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0)) + x.a, x.b = 0, -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0)) def test_unsigned(self): for c_typ in unsigned_int_types: - class X(Structure): - _fields_ = [("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)) - - x = X() - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) - x.a = -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0)) - x.a, x.b = 0, -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + class X(Structure): + _fields_ = [("a", c_typ, 3), + ("b", c_typ, 3), + ("c", c_typ, 1)] + self.check_struct(X) + self.assertEqual(sizeof(X), sizeof(c_typ)) + + x = X() + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) + x.a = -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0)) + x.a, x.b = 0, -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) def fail_fields(self, *fields): - return self.get_except(type(Structure), "X", (), - {"_fields_": fields}) + for layout in "ms", "gcc-sysv": + with self.subTest(layout=layout): + return self.get_except(type(Structure), "X", (), + {"_fields_": fields, "layout": layout}) def test_nonint_types(self): # bit fields are not allowed on non-integer types. @@ -136,8 +212,16 @@ def test_nonint_types(self): result = self.fail_fields(("a", c_char, 1)) self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char')) - class Dummy(Structure): + class Empty(Structure): _fields_ = [] + self.check_struct(Empty) + + result = self.fail_fields(("a", Empty, 1)) + self.assertEqual(result, (ValueError, "number of bits invalid for bit field 'a'")) + + class Dummy(Structure): + _fields_ = [("x", c_int)] + self.check_struct(Dummy) result = self.fail_fields(("a", Dummy, 1)) self.assertEqual(result, (TypeError, 'bit fields not allowed for type Dummy')) @@ -149,28 +233,37 @@ def test_c_wchar(self): def test_single_bitfield_size(self): for c_typ in int_types: - result = self.fail_fields(("a", c_typ, -1)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) - - result = self.fail_fields(("a", c_typ, 0)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) - - class X(Structure): - _fields_ = [("a", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)) - - class X(Structure): - _fields_ = [("a", c_typ, sizeof(c_typ)*8)] - self.assertEqual(sizeof(X), sizeof(c_typ)) - - result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + result = self.fail_fields(("a", c_typ, -1)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) + + result = self.fail_fields(("a", c_typ, 0)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) + + class X(Structure): + _fields_ = [("a", c_typ, 1)] + self.check_struct(X) + self.assertEqual(sizeof(X), sizeof(c_typ)) + + class X(Structure): + _fields_ = [("a", c_typ, sizeof(c_typ)*8)] + self.check_struct(X) + self.assertEqual(sizeof(X), sizeof(c_typ)) + + result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) def test_multi_bitfields_size(self): class X(Structure): _fields_ = [("a", c_short, 1), ("b", c_short, 14), ("c", c_short, 1)] + self.check_struct(X) self.assertEqual(sizeof(X), sizeof(c_short)) class X(Structure): @@ -178,6 +271,7 @@ class X(Structure): ("a1", c_short), ("b", c_short, 14), ("c", c_short, 1)] + self.check_struct(X) self.assertEqual(sizeof(X), sizeof(c_short)*3) self.assertEqual(X.a.offset, 0) self.assertEqual(X.a1.offset, sizeof(c_short)) @@ -188,6 +282,7 @@ class X(Structure): _fields_ = [("a", c_short, 3), ("b", c_short, 14), ("c", c_short, 14)] + self.check_struct(X) self.assertEqual(sizeof(X), sizeof(c_short)*3) self.assertEqual(X.a.offset, sizeof(c_short)*0) self.assertEqual(X.b.offset, sizeof(c_short)*1) @@ -203,6 +298,7 @@ def test_mixed_1(self): class X(Structure): _fields_ = [("a", c_byte, 4), ("b", c_int, 4)] + self.check_struct(X) if os.name == "nt": self.assertEqual(sizeof(X), sizeof(c_int)*2) else: @@ -212,12 +308,14 @@ def test_mixed_2(self): class X(Structure): _fields_ = [("a", c_byte, 4), ("b", c_int, 32)] + self.check_struct(X) self.assertEqual(sizeof(X), alignment(c_int)+sizeof(c_int)) def test_mixed_3(self): class X(Structure): _fields_ = [("a", c_byte, 4), ("b", c_ubyte, 4)] + self.check_struct(X) self.assertEqual(sizeof(X), sizeof(c_byte)) def test_mixed_4(self): @@ -228,6 +326,7 @@ class X(Structure): ("d", c_short, 4), ("e", c_short, 4), ("f", c_int, 24)] + self.check_struct(X) # MSVC does NOT combine c_short and c_int into one field, GCC # does (unless GCC is run with '-mms-bitfields' which # produces code compatible with MSVC). @@ -236,6 +335,177 @@ class X(Structure): else: self.assertEqual(sizeof(X), sizeof(c_int) * 2) + def test_mixed_5(self): + class X(Structure): + _fields_ = [ + ('A', c_uint, 1), + ('B', c_ushort, 16)] + self.check_struct(X) + a = X() + a.A = 0 + a.B = 1 + self.assertEqual(1, a.B) + + def test_mixed_6(self): + class X(Structure): + _fields_ = [ + ('A', c_ulonglong, 1), + ('B', c_uint, 32)] + self.check_struct(X) + a = X() + a.A = 0 + a.B = 1 + self.assertEqual(1, a.B) + + @unittest.skipIf(sizeof(c_uint64) != alignment(c_uint64), + 'assumes size=alignment') + def test_mixed_7(self): + class X(Structure): + _fields_ = [ + ("A", c_uint32), + ('B', c_uint32, 20), + ('C', c_uint64, 24)] + self.check_struct(X) + self.assertEqual(16, sizeof(X)) + + def test_mixed_8(self): + class Foo(Structure): + _fields_ = [ + ("A", c_uint32), + ("B", c_uint32, 32), + ("C", c_ulonglong, 1), + ] + self.check_struct(Foo) + + class Bar(Structure): + _fields_ = [ + ("A", c_uint32), + ("B", c_uint32), + ("C", c_ulonglong, 1), + ] + self.check_struct(Bar) + self.assertEqual(sizeof(Foo), sizeof(Bar)) + + def test_mixed_9(self): + class X(Structure): + _fields_ = [ + ("A", c_uint8), + ("B", c_uint32, 1), + ] + self.check_struct(X) + if sys.platform == 'win32': + self.assertEqual(8, sizeof(X)) + else: + self.assertEqual(4, sizeof(X)) + + @unittest.skipIf(sizeof(c_uint64) != alignment(c_uint64), + 'assumes size=alignment') + def test_mixed_10(self): + class X(Structure): + _fields_ = [ + ("A", c_uint32, 1), + ("B", c_uint64, 1), + ] + self.check_struct(X) + if sys.platform == 'win32': + self.assertEqual(8, alignment(X)) + self.assertEqual(16, sizeof(X)) + else: + self.assertEqual(8, alignment(X)) + self.assertEqual(8, sizeof(X)) + + def test_gh_95496(self): + for field_width in range(1, 33): + class TestStruct(Structure): + _fields_ = [ + ("Field1", c_uint32, field_width), + ("Field2", c_uint8, 8) + ] + self.check_struct(TestStruct) + + cmd = TestStruct() + cmd.Field2 = 1 + self.assertEqual(1, cmd.Field2) + + def test_gh_84039(self): + class Bad(Structure): + _pack_ = 1 + _layout_ = "ms" + _fields_ = [ + ("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12), + ] + + class GoodA(Structure): + _pack_ = 1 + _layout_ = "ms" + _fields_ = [ + ("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ] + + + class Good(Structure): + _pack_ = 1 + _layout_ = "ms" + _fields_ = [ + ("a", GoodA), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12), + ] + self.check_struct(Bad) + self.check_struct(GoodA) + self.check_struct(Good) + + self.assertEqual(3, sizeof(Bad)) + self.assertEqual(3, sizeof(Good)) + + def test_gh_73939(self): + class MyStructure(Structure): + _pack_ = 1 + _layout_ = "ms" + _fields_ = [ + ("P", c_uint16), + ("L", c_uint16, 9), + ("Pro", c_uint16, 1), + ("G", c_uint16, 1), + ("IB", c_uint16, 1), + ("IR", c_uint16, 1), + ("R", c_uint16, 3), + ("T", c_uint32, 10), + ("C", c_uint32, 20), + ("R2", c_uint32, 2) + ] + self.check_struct(MyStructure) + self.assertEqual(8, sizeof(MyStructure)) + + def test_gh_86098(self): + class X(Structure): + _fields_ = [ + ("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16) + ] + self.check_struct(X) + if sys.platform == 'win32': + self.assertEqual(8, sizeof(X)) + else: + self.assertEqual(4, sizeof(X)) + def test_anon_bitfields(self): # anonymous bit-fields gave a strange error message class X(Structure): @@ -245,9 +515,13 @@ class Y(Structure): _anonymous_ = ["_"] _fields_ = [("_", X)] + self.check_struct(X) + self.check_struct(Y) + def test_uint32(self): class X(Structure): _fields_ = [("a", c_uint32, 32)] + self.check_struct(X) x = X() x.a = 10 self.assertEqual(x.a, 10) @@ -257,6 +531,7 @@ class X(Structure): def test_uint64(self): class X(Structure): _fields_ = [("a", c_uint64, 64)] + self.check_struct(X) x = X() x.a = 10 self.assertEqual(x.a, 10) @@ -269,6 +544,7 @@ class Little(LittleEndianStructure): _fields_ = [("a", c_uint32, 24), ("b", c_uint32, 4), ("c", c_uint32, 4)] + self.check_struct(Little) b = bytearray(4) x = Little.from_buffer(b) x.a = 0xabcdef @@ -282,6 +558,7 @@ class Big(BigEndianStructure): _fields_ = [("a", c_uint32, 24), ("b", c_uint32, 4), ("c", c_uint32, 4)] + self.check_struct(Big) b = bytearray(4) x = Big.from_buffer(b) x.a = 0xabcdef @@ -289,6 +566,22 @@ class Big(BigEndianStructure): x.c = 2 self.assertEqual(b, b'\xab\xcd\xef\x12') + def test_union_bitfield(self): + class BitfieldUnion(Union): + _fields_ = [("a", c_uint32, 1), + ("b", c_uint32, 2), + ("c", c_uint32, 3)] + self.check_union(BitfieldUnion) + self.assertEqual(sizeof(BitfieldUnion), 4) + b = bytearray(4) + x = BitfieldUnion.from_buffer(b) + x.a = 1 + self.assertEqual(int.from_bytes(b).bit_count(), 1) + x.b = 3 + self.assertEqual(int.from_bytes(b).bit_count(), 2) + x.c = 7 + self.assertEqual(int.from_bytes(b).bit_count(), 3) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_bytes.py b/Lib/test/test_ctypes/test_bytes.py index fa11e1bbd49..0e7f81b9482 100644 --- a/Lib/test/test_ctypes/test_bytes.py +++ b/Lib/test/test_ctypes/test_bytes.py @@ -3,9 +3,10 @@ import unittest from _ctypes import _SimpleCData from ctypes import Structure, c_char, c_char_p, c_wchar, c_wchar_p +from ._support import StructCheckMixin -class BytesTest(unittest.TestCase): +class BytesTest(unittest.TestCase, StructCheckMixin): def test_c_char(self): x = c_char(b"x") self.assertRaises(TypeError, c_char, "x") @@ -40,6 +41,7 @@ def test_c_wchar_p(self): def test_struct(self): class X(Structure): _fields_ = [("a", c_char * 3)] + self.check_struct(X) x = X(b"abc") self.assertRaises(TypeError, X, "abc") @@ -49,6 +51,7 @@ class X(Structure): def test_struct_W(self): class X(Structure): _fields_ = [("a", c_wchar * 3)] + self.check_struct(X) x = X("abc") self.assertRaises(TypeError, X, b"abc") diff --git a/Lib/test/test_ctypes/test_byteswap.py b/Lib/test/test_ctypes/test_byteswap.py index 78eff0392c4..ea5951603f9 100644 --- a/Lib/test/test_ctypes/test_byteswap.py +++ b/Lib/test/test_ctypes/test_byteswap.py @@ -11,6 +11,7 @@ c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_uint32, c_float, c_double) +from ._support import StructCheckMixin def bin(s): @@ -24,15 +25,17 @@ def bin(s): # # For Structures and Unions, these types are created on demand. -class Test(unittest.TestCase): +class Test(unittest.TestCase, StructCheckMixin): def test_slots(self): class BigPoint(BigEndianStructure): __slots__ = () _fields_ = [("x", c_int), ("y", c_int)] + self.check_struct(BigPoint) class LowPoint(LittleEndianStructure): __slots__ = () _fields_ = [("x", c_int), ("y", c_int)] + self.check_struct(LowPoint) big = BigPoint() little = LowPoint() @@ -200,6 +203,7 @@ def test_struct_fields_unsupported_byte_order(self): with self.assertRaises(TypeError): class T(BigEndianStructure if sys.byteorder == "little" else LittleEndianStructure): _fields_ = fields + [("x", typ)] + self.check_struct(T) def test_struct_struct(self): @@ -219,14 +223,15 @@ def test_struct_struct(self): class NestedStructure(nested): _fields_ = [("x", c_uint32), ("y", c_uint32)] + self.check_struct(NestedStructure) class TestStructure(parent): _fields_ = [("point", NestedStructure)] + self.check_struct(TestStructure) self.assertEqual(len(data), sizeof(TestStructure)) ptr = POINTER(TestStructure) s = cast(data, ptr)[0] - del ctypes._pointer_type_cache[TestStructure] self.assertEqual(s.point.x, 1) self.assertEqual(s.point.y, 2) @@ -248,6 +253,7 @@ class S(base): ("h", c_short), ("i", c_int), ("d", c_double)] + self.check_struct(S) s1 = S(0x12, 0x1234, 0x12345678, 3.14) s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14) @@ -263,6 +269,7 @@ def test_unaligned_nonnative_struct_fields(self): class S(base): _pack_ = 1 + _layout_ = "ms" _fields_ = [("b", c_byte), ("h", c_short), @@ -271,6 +278,7 @@ class S(base): ("_2", c_byte), ("d", c_double)] + self.check_struct(S) s1 = S() s1.b = 0x12 @@ -289,6 +297,7 @@ def test_unaligned_native_struct_fields(self): class S(Structure): _pack_ = 1 + _layout_ = "ms" _fields_ = [("b", c_byte), ("h", c_short), @@ -298,6 +307,7 @@ class S(Structure): ("_2", c_byte), ("d", c_double)] + self.check_struct(S) s1 = S() s1.b = 0x12 @@ -334,6 +344,7 @@ def test_union_fields_unsupported_byte_order(self): with self.assertRaises(TypeError): class T(BigEndianUnion if sys.byteorder == "little" else LittleEndianUnion): _fields_ = fields + [("x", typ)] + self.check_union(T) def test_union_struct(self): # nested structures in unions with different byteorders @@ -352,14 +363,15 @@ def test_union_struct(self): class NestedStructure(nested): _fields_ = [("x", c_uint32), ("y", c_uint32)] + self.check_struct(NestedStructure) class TestUnion(parent): _fields_ = [("point", NestedStructure)] + self.check_union(TestUnion) self.assertEqual(len(data), sizeof(TestUnion)) ptr = POINTER(TestUnion) s = cast(data, ptr)[0] - del ctypes._pointer_type_cache[TestUnion] self.assertEqual(s.point.x, 1) self.assertEqual(s.point.y, 2) @@ -374,12 +386,15 @@ def test_build_struct_union_opposite_system_byteorder(self): class S1(_Structure): _fields_ = [("a", c_byte), ("b", c_byte)] + self.check_struct(S1) class U1(_Union): _fields_ = [("s1", S1), ("ab", c_short)] + self.check_union(U1) class S2(_Structure): _fields_ = [("u1", U1), ("c", c_byte)] + self.check_struct(S2) if __name__ == "__main__": diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py index eb77d6d7782..fd261acf497 100644 --- a/Lib/test/test_ctypes/test_c_simple_type_meta.py +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -1,15 +1,15 @@ import unittest +from test.support import MS_WINDOWS import ctypes -from ctypes import POINTER, c_void_p +from ctypes import POINTER, Structure, c_void_p -from ._support import PyCSimpleType +from ._support import PyCSimpleType, PyCPointerType, PyCStructType -class PyCSimpleTypeAsMetaclassTest(unittest.TestCase): - def tearDown(self): - # to not leak references, we must clean _pointer_type_cache - ctypes._reset_cache() +def set_non_ctypes_pointer_type(cls, pointer_type): + cls.__pointer_type__ = pointer_type +class PyCSimpleTypeAsMetaclassTest(unittest.TestCase): def test_creating_pointer_in_dunder_new_1(self): # Test metaclass whose instances are C types; when the type is # created it automatically creates a pointer type for itself. @@ -35,7 +35,7 @@ def __new__(cls, name, bases, namespace): else: ptr_bases = (self, POINTER(bases[0])) p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) - ctypes._pointer_type_cache[self] = p + set_non_ctypes_pointer_type(self, p) return self class p_meta(PyCSimpleType, ct_meta): @@ -44,19 +44,35 @@ class p_meta(PyCSimpleType, ct_meta): class PtrBase(c_void_p, metaclass=p_meta): pass + ptr_base_pointer = POINTER(PtrBase) + class CtBase(object, metaclass=ct_meta): pass + ct_base_pointer = POINTER(CtBase) + class Sub(CtBase): pass + sub_pointer = POINTER(Sub) + class Sub2(Sub): pass + sub2_pointer = POINTER(Sub2) + + self.assertIsNot(ptr_base_pointer, ct_base_pointer) + self.assertIsNot(ct_base_pointer, sub_pointer) + self.assertIsNot(sub_pointer, sub2_pointer) + self.assertIsInstance(POINTER(Sub2), p_meta) - self.assertTrue(issubclass(POINTER(Sub2), Sub2)) - self.assertTrue(issubclass(POINTER(Sub2), POINTER(Sub))) - self.assertTrue(issubclass(POINTER(Sub), POINTER(CtBase))) + self.assertIsSubclass(POINTER(Sub2), Sub2) + self.assertIsSubclass(POINTER(Sub2), POINTER(Sub)) + self.assertIsSubclass(POINTER(Sub), POINTER(CtBase)) + + self.assertIs(POINTER(Sub2), sub2_pointer) + self.assertIs(POINTER(Sub), sub_pointer) + self.assertIs(POINTER(CtBase), ct_base_pointer) def test_creating_pointer_in_dunder_new_2(self): # A simpler variant of the above, used in `CoClass` of the `comtypes` @@ -68,7 +84,7 @@ def __new__(cls, name, bases, namespace): if isinstance(self, p_meta): return self p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) - ctypes._pointer_type_cache[self] = p + set_non_ctypes_pointer_type(self, p) return self class p_meta(PyCSimpleType, ct_meta): @@ -77,14 +93,26 @@ class p_meta(PyCSimpleType, ct_meta): class Core(object): pass + with self.assertRaisesRegex(TypeError, "must have storage info"): + POINTER(Core) + class CtBase(Core, metaclass=ct_meta): pass + ct_base_pointer = POINTER(CtBase) + class Sub(CtBase): pass + sub_pointer = POINTER(Sub) + + self.assertIsNot(ct_base_pointer, sub_pointer) + self.assertIsInstance(POINTER(Sub), p_meta) - self.assertTrue(issubclass(POINTER(Sub), Sub)) + self.assertIsSubclass(POINTER(Sub), Sub) + + self.assertIs(POINTER(Sub), sub_pointer) + self.assertIs(POINTER(CtBase), ct_base_pointer) def test_creating_pointer_in_dunder_init_1(self): class ct_meta(type): @@ -102,7 +130,7 @@ def __init__(self, name, bases, namespace): else: ptr_bases = (self, POINTER(bases[0])) p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) - ctypes._pointer_type_cache[self] = p + set_non_ctypes_pointer_type(self, p) class p_meta(PyCSimpleType, ct_meta): pass @@ -110,19 +138,36 @@ class p_meta(PyCSimpleType, ct_meta): class PtrBase(c_void_p, metaclass=p_meta): pass + ptr_base_pointer = POINTER(PtrBase) + class CtBase(object, metaclass=ct_meta): pass + ct_base_pointer = POINTER(CtBase) + class Sub(CtBase): pass + sub_pointer = POINTER(Sub) + class Sub2(Sub): pass + sub2_pointer = POINTER(Sub2) + + self.assertIsNot(ptr_base_pointer, ct_base_pointer) + self.assertIsNot(ct_base_pointer, sub_pointer) + self.assertIsNot(sub_pointer, sub2_pointer) + self.assertIsInstance(POINTER(Sub2), p_meta) - self.assertTrue(issubclass(POINTER(Sub2), Sub2)) - self.assertTrue(issubclass(POINTER(Sub2), POINTER(Sub))) - self.assertTrue(issubclass(POINTER(Sub), POINTER(CtBase))) + self.assertIsSubclass(POINTER(Sub2), Sub2) + self.assertIsSubclass(POINTER(Sub2), POINTER(Sub)) + self.assertIsSubclass(POINTER(Sub), POINTER(CtBase)) + + self.assertIs(POINTER(PtrBase), ptr_base_pointer) + self.assertIs(POINTER(CtBase), ct_base_pointer) + self.assertIs(POINTER(Sub), sub_pointer) + self.assertIs(POINTER(Sub2), sub2_pointer) def test_creating_pointer_in_dunder_init_2(self): class ct_meta(type): @@ -134,7 +179,7 @@ def __init__(self, name, bases, namespace): if isinstance(self, p_meta): return p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) - ctypes._pointer_type_cache[self] = p + set_non_ctypes_pointer_type(self, p) class p_meta(PyCSimpleType, ct_meta): pass @@ -145,8 +190,195 @@ class Core(object): class CtBase(Core, metaclass=ct_meta): pass + ct_base_pointer = POINTER(CtBase) + class Sub(CtBase): pass + sub_pointer = POINTER(Sub) + + self.assertIsNot(ct_base_pointer, sub_pointer) + self.assertIsInstance(POINTER(Sub), p_meta) - self.assertTrue(issubclass(POINTER(Sub), Sub)) + self.assertIsSubclass(POINTER(Sub), Sub) + + self.assertIs(POINTER(CtBase), ct_base_pointer) + self.assertIs(POINTER(Sub), sub_pointer) + + def test_bad_type_message(self): + """Verify the error message that lists all available type codes""" + # (The string is generated at runtime, so this checks the underlying + # set of types as well as correct construction of the string.) + with self.assertRaises(AttributeError) as cm: + class F(metaclass=PyCSimpleType): + _type_ = "\0" + message = str(cm.exception) + expected_type_chars = list('cbBhHiIlLdDFGfuzZqQPXOv?g') + if not hasattr(ctypes, 'c_float_complex'): + expected_type_chars.remove('F') + expected_type_chars.remove('D') + expected_type_chars.remove('G') + if not MS_WINDOWS: + expected_type_chars.remove('X') + self.assertIn("'" + ''.join(expected_type_chars) + "'", message) + + def test_creating_pointer_in_dunder_init_3(self): + """Check if interfcase subclasses properly creates according internal + pointer types. But not the same as external pointer types. + """ + + class StructureMeta(PyCStructType): + def __new__(cls, name, bases, dct, /, create_pointer_type=True): + assert len(bases) == 1, bases + return super().__new__(cls, name, bases, dct) + + def __init__(self, name, bases, dct, /, create_pointer_type=True): + + super().__init__(name, bases, dct) + if create_pointer_type: + p_bases = (POINTER(bases[0]),) + ns = {'_type_': self} + internal_pointer_type = PointerMeta(f"p{name}", p_bases, ns) + assert isinstance(internal_pointer_type, PyCPointerType) + assert self.__pointer_type__ is internal_pointer_type + + class PointerMeta(PyCPointerType): + def __new__(cls, name, bases, dct): + target = dct.get('_type_', None) + if target is None: + + # Create corresponding interface type and then set it as target + target = StructureMeta( + f"_{name}_", + (bases[0]._type_,), + {}, + create_pointer_type=False + ) + dct['_type_'] = target + + pointer_type = super().__new__(cls, name, bases, dct) + assert not hasattr(target, '__pointer_type__') + + return pointer_type + + def __init__(self, name, bases, dct, /, create_pointer_type=True): + target = dct.get('_type_', None) + assert not hasattr(target, '__pointer_type__') + super().__init__(name, bases, dct) + assert target.__pointer_type__ is self + + + class Interface(Structure, metaclass=StructureMeta, create_pointer_type=False): + pass + + class pInterface(POINTER(c_void_p), metaclass=PointerMeta): + _type_ = Interface + + class IUnknown(Interface): + pass + + class pIUnknown(pInterface): + pass + + self.assertTrue(issubclass(POINTER(IUnknown), pInterface)) + + self.assertIs(POINTER(Interface), pInterface) + self.assertIsNot(POINTER(IUnknown), pIUnknown) + + def test_creating_pointer_in_dunder_init_4(self): + """Check if interfcase subclasses properly creates according internal + pointer types, the same as external pointer types. + """ + class StructureMeta(PyCStructType): + def __new__(cls, name, bases, dct, /, create_pointer_type=True): + assert len(bases) == 1, bases + + return super().__new__(cls, name, bases, dct) + + def __init__(self, name, bases, dct, /, create_pointer_type=True): + + super().__init__(name, bases, dct) + if create_pointer_type: + p_bases = (POINTER(bases[0]),) + ns = {'_type_': self} + internal_pointer_type = PointerMeta(f"p{name}", p_bases, ns) + assert isinstance(internal_pointer_type, PyCPointerType) + assert self.__pointer_type__ is internal_pointer_type + + class PointerMeta(PyCPointerType): + def __new__(cls, name, bases, dct): + target = dct.get('_type_', None) + assert target is not None + pointer_type = getattr(target, '__pointer_type__', None) + + if pointer_type is None: + pointer_type = super().__new__(cls, name, bases, dct) + + return pointer_type + + def __init__(self, name, bases, dct, /, create_pointer_type=True): + target = dct.get('_type_', None) + if not hasattr(target, '__pointer_type__'): + # target.__pointer_type__ was created by super().__new__ + super().__init__(name, bases, dct) + + assert target.__pointer_type__ is self + + + class Interface(Structure, metaclass=StructureMeta, create_pointer_type=False): + pass + + class pInterface(POINTER(c_void_p), metaclass=PointerMeta): + _type_ = Interface + + class IUnknown(Interface): + pass + + class pIUnknown(pInterface): + _type_ = IUnknown + + self.assertTrue(issubclass(POINTER(IUnknown), pInterface)) + + self.assertIs(POINTER(Interface), pInterface) + self.assertIs(POINTER(IUnknown), pIUnknown) + + def test_custom_pointer_cache_for_ctypes_type1(self): + # Check if PyCPointerType.__init__() caches a pointer type + # customized in the metatype's __new__(). + class PointerMeta(PyCPointerType): + def __new__(cls, name, bases, namespace): + namespace["_type_"] = C + return super().__new__(cls, name, bases, namespace) + + def __init__(self, name, bases, namespace): + assert not hasattr(C, '__pointer_type__') + super().__init__(name, bases, namespace) + assert C.__pointer_type__ is self + + class C(c_void_p): # ctypes type + pass + + class P(ctypes._Pointer, metaclass=PointerMeta): + pass + + self.assertIs(P._type_, C) + self.assertIs(P, POINTER(C)) + + def test_custom_pointer_cache_for_ctypes_type2(self): + # Check if PyCPointerType.__init__() caches a pointer type + # customized in the metatype's __init__(). + class PointerMeta(PyCPointerType): + def __init__(self, name, bases, namespace): + self._type_ = namespace["_type_"] = C + assert not hasattr(C, '__pointer_type__') + super().__init__(name, bases, namespace) + assert C.__pointer_type__ is self + + class C(c_void_p): # ctypes type + pass + + class P(ctypes._Pointer, metaclass=PointerMeta): + pass + + self.assertIs(P._type_, C) + self.assertIs(P, POINTER(C)) diff --git a/Lib/test/test_ctypes/test_callbacks.py b/Lib/test/test_ctypes/test_callbacks.py index 8f483dfe1db..6c7c2e52707 100644 --- a/Lib/test/test_ctypes/test_callbacks.py +++ b/Lib/test/test_ctypes/test_callbacks.py @@ -324,7 +324,7 @@ def func(): self.assertIsInstance(cm.unraisable.exc_value, TypeError) self.assertEqual(cm.unraisable.err_msg, - f"Exception ignored on converting result " + f"Exception ignored while converting result " f"of ctypes callback function {func!r}") self.assertIsNone(cm.unraisable.object) diff --git a/Lib/test/test_ctypes/test_cast.py b/Lib/test/test_ctypes/test_cast.py index db6bdc75eff..604f44f03d6 100644 --- a/Lib/test/test_ctypes/test_cast.py +++ b/Lib/test/test_ctypes/test_cast.py @@ -32,8 +32,6 @@ def test_address2pointer(self): ptr = cast(address, POINTER(c_int)) self.assertEqual([ptr[i] for i in range(3)], [42, 17, 2]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_p2a_objects(self): array = (c_char_p * 5)() self.assertEqual(array._objects, None) diff --git a/Lib/test/test_ctypes/test_cfuncs.py b/Lib/test/test_ctypes/test_cfuncs.py index 48330c4b0a7..937be8eaa95 100644 --- a/Lib/test/test_ctypes/test_cfuncs.py +++ b/Lib/test/test_ctypes/test_cfuncs.py @@ -5,7 +5,8 @@ c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double, c_longdouble) -from test.support import import_helper +from test import support +from test.support import import_helper, threading_helper _ctypes_test = import_helper.import_module("_ctypes_test") @@ -13,9 +14,10 @@ class CFunctions(unittest.TestCase): _dll = CDLL(_ctypes_test.__file__) def S(self): - return c_longlong.in_dll(self._dll, "last_tf_arg_s").value + return _ctypes_test.get_last_tf_arg_s() + def U(self): - return c_ulonglong.in_dll(self._dll, "last_tf_arg_u").value + return _ctypes_test.get_last_tf_arg_u() def test_byte(self): self._dll.tf_b.restype = c_byte @@ -191,6 +193,23 @@ def test_void(self): self.assertEqual(self._dll.tv_i(-42), None) self.assertEqual(self.S(), -42) + @threading_helper.requires_working_threading() + @support.requires_resource("cpu") + @unittest.skipUnless(support.Py_GIL_DISABLED, "only meaningful on free-threading") + def test_thread_safety(self): + from threading import Thread + + def concurrent(): + for _ in range(100): + self._dll.tf_b.restype = c_byte + self._dll.tf_b.argtypes = (c_byte,) + + with threading_helper.catch_threading_exception() as exc: + with threading_helper.start_threads((Thread(target=concurrent) for _ in range(10))): + pass + + self.assertIsNone(exc.exc_value) + # The following repeats the above tests with stdcall functions (where # they are available) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index cd87bad3825..ea2d97d9000 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -32,6 +32,7 @@ @unittest.skipUnless(sys.platform.startswith('linux'), 'test requires GNU IFUNC support') +@unittest.skipIf(test.support.linked_to_musl(), "Requires glibc") class TestNullDlsym(unittest.TestCase): """GH-126554: Ensure that we catch NULL dlsym return values @@ -54,7 +55,8 @@ class TestNullDlsym(unittest.TestCase): this 'dlsym returned NULL -> throw Error' rule. """ - @unittest.expectedFailure # TODO: RUSTPYTHON + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_null_dlsym(self): import subprocess import tempfile @@ -121,7 +123,7 @@ def test_null_dlsym(self): # Assert that the IFUNC was called self.assertEqual(os.read(pipe_r, 2), b'OK') - +@test.support.thread_unsafe('setlocale is not thread-safe') @unittest.skipUnless(os.name != 'nt', 'test requires dlerror() calls') class TestLocalization(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_dllist.py b/Lib/test/test_ctypes/test_dllist.py new file mode 100644 index 00000000000..0e7c65127f6 --- /dev/null +++ b/Lib/test/test_ctypes/test_dllist.py @@ -0,0 +1,63 @@ +import os +import sys +import unittest +from ctypes import CDLL +import ctypes.util +from test.support import import_helper + + +WINDOWS = os.name == "nt" +APPLE = sys.platform in {"darwin", "ios", "tvos", "watchos"} + +if WINDOWS: + KNOWN_LIBRARIES = ["KERNEL32.DLL"] +elif APPLE: + KNOWN_LIBRARIES = ["libSystem.B.dylib"] +else: + # trickier than it seems, because libc may not be present + # on musl systems, and sometimes goes by different names. + # However, ctypes itself loads libffi + KNOWN_LIBRARIES = ["libc.so", "libffi.so"] + + +@unittest.skipUnless( + hasattr(ctypes.util, "dllist"), + "ctypes.util.dllist is not available on this platform", +) +class ListSharedLibraries(unittest.TestCase): + + # TODO: RUSTPYTHON + @unittest.skipIf(not APPLE, "TODO: RUSTPYTHON") + def test_lists_system(self): + dlls = ctypes.util.dllist() + + self.assertGreater(len(dlls), 0, f"loaded={dlls}") + self.assertTrue( + any(lib in dll for dll in dlls for lib in KNOWN_LIBRARIES), f"loaded={dlls}" + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_lists_updates(self): + dlls = ctypes.util.dllist() + + # this test relies on being able to import a library which is + # not already loaded. + # If it is (e.g. by a previous test in the same process), we skip + if any("_ctypes_test" in dll for dll in dlls): + self.skipTest("Test library is already loaded") + + _ctypes_test = import_helper.import_module("_ctypes_test") + test_module = CDLL(_ctypes_test.__file__) + dlls2 = ctypes.util.dllist() + self.assertIsNotNone(dlls2) + + dlls1 = set(dlls) + dlls2 = set(dlls2) + + self.assertGreater(dlls2, dlls1, f"newly loaded libraries: {dlls2 - dlls1}") + self.assertTrue(any("_ctypes_test" in dll for dll in dlls2), f"loaded={dlls2}") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_ctypes/test_find.py b/Lib/test/test_ctypes/test_find.py index 85b28617d2d..8bc84c3d2ef 100644 --- a/Lib/test/test_ctypes/test_find.py +++ b/Lib/test/test_ctypes/test_find.py @@ -5,7 +5,7 @@ import unittest.mock from ctypes import CDLL, RTLD_GLOBAL from ctypes.util import find_library -from test.support import os_helper +from test.support import os_helper, thread_unsafe # On some systems, loading the OpenGL libraries needs the RTLD_GLOBAL mode. @@ -78,6 +78,7 @@ def test_shell_injection(self): @unittest.skipUnless(sys.platform.startswith('linux'), 'Test only valid for Linux') class FindLibraryLinux(unittest.TestCase): + @thread_unsafe('uses setenv') def test_find_on_libpath(self): import subprocess import tempfile @@ -152,5 +153,73 @@ def test_find(self): self.assertIsNone(find_library(name)) +@unittest.skipUnless(test.support.is_emscripten, + 'Test only valid for Emscripten') +class FindLibraryEmscripten(unittest.TestCase): + @classmethod + def setUpClass(cls): + import tempfile + + # A very simple wasm module + # In WAT format: (module) + cls.wasm_module = b'\x00asm\x01\x00\x00\x00\x00\x08\x04name\x02\x01\x00' + + cls.non_wasm_content = b'This is not a WASM file' + + cls.temp_dir = tempfile.mkdtemp() + cls.libdummy_so_path = os.path.join(cls.temp_dir, 'libdummy.so') + with open(cls.libdummy_so_path, 'wb') as f: + f.write(cls.wasm_module) + + cls.libother_wasm_path = os.path.join(cls.temp_dir, 'libother.wasm') + with open(cls.libother_wasm_path, 'wb') as f: + f.write(cls.wasm_module) + + cls.libnowasm_so_path = os.path.join(cls.temp_dir, 'libnowasm.so') + with open(cls.libnowasm_so_path, 'wb') as f: + f.write(cls.non_wasm_content) + + @classmethod + def tearDownClass(cls): + import shutil + shutil.rmtree(cls.temp_dir) + + def test_find_wasm_file_with_so_extension(self): + with os_helper.EnvironmentVarGuard() as env: + env.set('LD_LIBRARY_PATH', self.temp_dir) + result = find_library('dummy') + self.assertEqual(result, self.libdummy_so_path) + def test_find_wasm_file_with_wasm_extension(self): + with os_helper.EnvironmentVarGuard() as env: + env.set('LD_LIBRARY_PATH', self.temp_dir) + result = find_library('other') + self.assertEqual(result, self.libother_wasm_path) + + def test_ignore_non_wasm_file(self): + with os_helper.EnvironmentVarGuard() as env: + env.set('LD_LIBRARY_PATH', self.temp_dir) + result = find_library('nowasm') + self.assertIsNone(result) + + def test_find_nothing_without_ld_library_path(self): + with os_helper.EnvironmentVarGuard() as env: + if 'LD_LIBRARY_PATH' in env: + del env['LD_LIBRARY_PATH'] + result = find_library('dummy') + self.assertIsNone(result) + result = find_library('other') + self.assertIsNone(result) + + def test_find_nothing_with_wrong_ld_library_path(self): + import tempfile + with tempfile.TemporaryDirectory() as empty_dir: + with os_helper.EnvironmentVarGuard() as env: + env.set('LD_LIBRARY_PATH', empty_dir) + result = find_library('dummy') + self.assertIsNone(result) + result = find_library('other') + self.assertIsNone(result) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_funcptr.py b/Lib/test/test_ctypes/test_funcptr.py index 8362fb16d94..be641da30ea 100644 --- a/Lib/test/test_ctypes/test_funcptr.py +++ b/Lib/test/test_ctypes/test_funcptr.py @@ -5,7 +5,7 @@ from test.support import import_helper _ctypes_test = import_helper.import_module("_ctypes_test") from ._support import (_CData, PyCFuncPtrType, Py_TPFLAGS_DISALLOW_INSTANTIATION, - Py_TPFLAGS_IMMUTABLETYPE) + Py_TPFLAGS_IMMUTABLETYPE, StructCheckMixin) try: @@ -17,7 +17,7 @@ lib = CDLL(_ctypes_test.__file__) -class CFuncPtrTestCase(unittest.TestCase): +class CFuncPtrTestCase(unittest.TestCase, StructCheckMixin): def test_inheritance_hierarchy(self): self.assertEqual(_CFuncPtr.mro(), [_CFuncPtr, _CData, object]) @@ -88,6 +88,7 @@ class WNDCLASS(Structure): ("hCursor", HCURSOR), ("lpszMenuName", LPCTSTR), ("lpszClassName", LPCTSTR)] + self.check_struct(WNDCLASS) wndclass = WNDCLASS() wndclass.lpfnWndProc = WNDPROC(wndproc) diff --git a/Lib/test/test_ctypes/test_functions.py b/Lib/test/test_ctypes/test_functions.py index 63e393f7b7c..3454b83d43e 100644 --- a/Lib/test/test_ctypes/test_functions.py +++ b/Lib/test/test_ctypes/test_functions.py @@ -2,7 +2,7 @@ import sys import unittest from ctypes import (CDLL, Structure, Array, CFUNCTYPE, - byref, POINTER, pointer, ArgumentError, + byref, POINTER, pointer, ArgumentError, sizeof, c_char, c_wchar, c_byte, c_char_p, c_wchar_p, c_short, c_int, c_long, c_longlong, c_void_p, c_float, c_double, c_longdouble) @@ -72,7 +72,8 @@ def callback(*args): self.assertEqual(str(cm.exception), "argument 1: TypeError: one character bytes, " - "bytearray or integer expected") + "bytearray, or an integer in range(256) expected, " + "not bytes of length 3") def test_wchar_parm(self): f = dll._testfunc_i_bhilfd @@ -84,14 +85,27 @@ def test_wchar_parm(self): with self.assertRaises(ArgumentError) as cm: f(1, 2, 3, 4, 5.0, 6.0) self.assertEqual(str(cm.exception), - "argument 2: TypeError: unicode string expected " - "instead of int instance") + "argument 2: TypeError: a unicode character expected, " + "not instance of int") with self.assertRaises(ArgumentError) as cm: f(1, "abc", 3, 4, 5.0, 6.0) self.assertEqual(str(cm.exception), - "argument 2: TypeError: one character unicode string " - "expected") + "argument 2: TypeError: a unicode character expected, " + "not a string of length 3") + + with self.assertRaises(ArgumentError) as cm: + f(1, "", 3, 4, 5.0, 6.0) + self.assertEqual(str(cm.exception), + "argument 2: TypeError: a unicode character expected, " + "not a string of length 0") + + if sizeof(c_wchar) < 4: + with self.assertRaises(ArgumentError) as cm: + f(1, "\U0001f40d", 3, 4, 5.0, 6.0) + self.assertEqual(str(cm.exception), + "argument 2: TypeError: the string '\\U0001f40d' " + "cannot be converted to a single wchar_t character") def test_c_char_p_parm(self): """Test the error message when converting an incompatible type to c_char_p.""" diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py new file mode 100644 index 00000000000..aa448fad5bb --- /dev/null +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -0,0 +1,761 @@ +"""Test CTypes structs, unions, bitfields against C equivalents. + +The types here are auto-converted to C source at +`Modules/_ctypes/_ctypes_test_generated.c.h`, which is compiled into +_ctypes_test. + +Run this module to regenerate the files: + +./python Lib/test/test_ctypes/test_generated_structs.py > Modules/_ctypes/_ctypes_test_generated.c.h +""" + +import unittest +from test.support import import_helper, verbose +import re +from dataclasses import dataclass +from functools import cached_property +import sys + +import ctypes +from ctypes import Structure, Union +from ctypes import sizeof, alignment, pointer, string_at +_ctypes_test = import_helper.import_module("_ctypes_test") + +from test.test_ctypes._support import StructCheckMixin + +# A 64-bit number where each nibble (hex digit) is different and +# has 2-3 bits set. +TEST_PATTERN = 0xae7596db + +# ctypes erases the difference between `c_int` and e.g.`c_int16`. +# To keep it, we'll use custom subclasses with the C name stashed in `_c_name`: +class c_bool(ctypes.c_bool): + _c_name = '_Bool' + +# To do it for all the other types, use some metaprogramming: +for c_name, ctypes_name in { + 'signed char': 'c_byte', + 'short': 'c_short', + 'int': 'c_int', + 'long': 'c_long', + 'long long': 'c_longlong', + 'unsigned char': 'c_ubyte', + 'unsigned short': 'c_ushort', + 'unsigned int': 'c_uint', + 'unsigned long': 'c_ulong', + 'unsigned long long': 'c_ulonglong', + **{f'{u}int{n}_t': f'c_{u}int{n}' + for u in ('', 'u') + for n in (8, 16, 32, 64)} +}.items(): + ctype = getattr(ctypes, ctypes_name) + newtype = type(ctypes_name, (ctype,), {'_c_name': c_name}) + globals()[ctypes_name] = newtype + + +# Register structs and unions to test + +TESTCASES = {} +def register(name=None, set_name=False): + def decorator(cls, name=name): + if name is None: + name = cls.__name__ + assert name.isascii() # will be used in _PyUnicode_EqualToASCIIString + assert name.isidentifier() # will be used as a C identifier + assert name not in TESTCASES + TESTCASES[name] = cls + if set_name: + cls.__name__ = name + return cls + return decorator + +@register() +class SingleInt(Structure): + _fields_ = [('a', c_int)] + +@register() +class SingleInt_Union(Union): + _fields_ = [('a', c_int)] + + +@register() +class SingleU32(Structure): + _fields_ = [('a', c_uint32)] + + +@register() +class SimpleStruct(Structure): + _fields_ = [('x', c_int32), ('y', c_int8), ('z', c_uint16)] + + +@register() +class SimpleUnion(Union): + _fields_ = [('x', c_int32), ('y', c_int8), ('z', c_uint16)] + + +@register() +class ManyTypes(Structure): + _fields_ = [ + ('i8', c_int8), ('u8', c_uint8), + ('i16', c_int16), ('u16', c_uint16), + ('i32', c_int32), ('u32', c_uint32), + ('i64', c_int64), ('u64', c_uint64), + ] + + +@register() +class ManyTypesU(Union): + _fields_ = [ + ('i8', c_int8), ('u8', c_uint8), + ('i16', c_int16), ('u16', c_uint16), + ('i32', c_int32), ('u32', c_uint32), + ('i64', c_int64), ('u64', c_uint64), + ] + + +@register() +class Nested(Structure): + _fields_ = [ + ('a', SimpleStruct), ('b', SimpleUnion), ('anon', SimpleStruct), + ] + _anonymous_ = ['anon'] + + +@register() +class Packed1(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 1 + _layout_ = 'ms' + + +@register() +class Packed2(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 2 + _layout_ = 'ms' + + +@register() +class Packed3(Structure): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 4 + _layout_ = 'ms' + + +@register() +class Packed4(Structure): + def _maybe_skip(): + # `_pack_` enables MSVC-style packing, but keeps platform-specific + # alignments. + # The C code we generate for GCC/clang currently uses + # `__attribute__((ms_struct))`, which activates MSVC layout *and* + # alignments, that is, sizeof(basic type) == alignment(basic type). + # On a Pentium, int64 is 32-bit aligned, so the two won't match. + # The expected behavior is instead tested in + # StructureTestCase.test_packed, over in test_structures.py. + if sizeof(c_int64) != alignment(c_int64): + raise unittest.SkipTest('cannot test on this platform') + + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 8 + _layout_ = 'ms' + +@register() +class X86_32EdgeCase(Structure): + # On a Pentium, long long (int64) is 32-bit aligned, + # so these are packed tightly. + _fields_ = [('a', c_int32), ('b', c_int64), ('c', c_int32)] + +@register() +class MSBitFieldExample(Structure): + # From https://learn.microsoft.com/en-us/cpp/c-language/c-bit-fields + _fields_ = [ + ('a', c_uint, 4), + ('b', c_uint, 5), + ('c', c_uint, 7)] + +@register() +class MSStraddlingExample(Structure): + # From https://learn.microsoft.com/en-us/cpp/c-language/c-bit-fields + _fields_ = [ + ('first', c_uint, 9), + ('second', c_uint, 7), + ('may_straddle', c_uint, 30), + ('last', c_uint, 18)] + +@register() +class IntBits(Structure): + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +@register() +class Bits(Structure): + _fields_ = [*IntBits._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +@register() +class IntBits_MSVC(Structure): + _layout_ = "ms" + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +@register() +class Bits_MSVC(Structure): + _layout_ = "ms" + _fields_ = [*IntBits_MSVC._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +# Skipped for now -- we don't always match the alignment +#@register() +class IntBits_Union(Union): + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +# Skipped for now -- we don't always match the alignment +#@register() +class BitsUnion(Union): + _fields_ = [*IntBits_Union._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +@register() +class I64Bits(Structure): + _fields_ = [("a", c_int64, 1), + ("b", c_int64, 62), + ("c", c_int64, 1)] + +@register() +class U64Bits(Structure): + _fields_ = [("a", c_uint64, 1), + ("b", c_uint64, 62), + ("c", c_uint64, 1)] + +for n in 8, 16, 32, 64: + for signedness in '', 'u': + ctype = globals()[f'c_{signedness}int{n}'] + + @register(f'Struct331_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 3), + ("b", ctype, 3), + ("c", ctype, 1)] + + @register(f'Struct1x1_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 1), + ("b", ctype, n-2), + ("c", ctype, 1)] + + @register(f'Struct1nx1_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 1), + ("full", ctype), + ("b", ctype, n-2), + ("c", ctype, 1)] + + @register(f'Struct3xx_{signedness}{n}', set_name=True) + class _cls(Structure): + _fields_ = [("a", ctype, 3), + ("b", ctype, n-2), + ("c", ctype, n-2)] + +@register() +class Mixed1(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int, 4)] + +@register() +class Mixed2(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int32, 32)] + +@register() +class Mixed3(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + +@register() +class Mixed4(Structure): + _fields_ = [("a", c_short, 4), + ("b", c_short, 4), + ("c", c_int, 24), + ("d", c_short, 4), + ("e", c_short, 4), + ("f", c_int, 24)] + +@register() +class Mixed5(Structure): + _fields_ = [('A', c_uint, 1), + ('B', c_ushort, 16)] + +@register() +class Mixed6(Structure): + _fields_ = [('A', c_ulonglong, 1), + ('B', c_uint, 32)] + +@register() +class Mixed7(Structure): + _fields_ = [("A", c_uint32), + ('B', c_uint32, 20), + ('C', c_uint64, 24)] + +@register() +class Mixed8_a(Structure): + _fields_ = [("A", c_uint32), + ("B", c_uint32, 32), + ("C", c_ulonglong, 1)] + +@register() +class Mixed8_b(Structure): + _fields_ = [("A", c_uint32), + ("B", c_uint32), + ("C", c_ulonglong, 1)] + +@register() +class Mixed9(Structure): + _fields_ = [("A", c_uint8), + ("B", c_uint32, 1)] + +@register() +class Mixed10(Structure): + _fields_ = [("A", c_uint32, 1), + ("B", c_uint64, 1)] + +@register() +class Example_gh_95496(Structure): + _fields_ = [("A", c_uint32, 1), + ("B", c_uint64, 1)] + +@register() +class Example_gh_84039_bad(Structure): + _pack_ = 1 + _layout_ = 'ms' + _fields_ = [("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12)] + +@register() +class Example_gh_84039_good_a(Structure): + _pack_ = 1 + _layout_ = 'ms' + _fields_ = [("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1)] + +@register() +class Example_gh_84039_good(Structure): + _pack_ = 1 + _layout_ = 'ms' + _fields_ = [("a", Example_gh_84039_good_a), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12)] + +@register() +class Example_gh_73939(Structure): + _pack_ = 1 + _layout_ = 'ms' + _fields_ = [("P", c_uint16), + ("L", c_uint16, 9), + ("Pro", c_uint16, 1), + ("G", c_uint16, 1), + ("IB", c_uint16, 1), + ("IR", c_uint16, 1), + ("R", c_uint16, 3), + ("T", c_uint32, 10), + ("C", c_uint32, 20), + ("R2", c_uint32, 2)] + +@register() +class Example_gh_86098(Structure): + _fields_ = [("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16)] + +@register() +class Example_gh_86098_pack(Structure): + _pack_ = 1 + _layout_ = 'ms' + _fields_ = [("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16)] + +@register() +class AnonBitfields(Structure): + class X(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + _anonymous_ = ["_"] + _fields_ = [("_", X), ('y', c_byte)] + + +class GeneratedTest(unittest.TestCase, StructCheckMixin): + def test_generated_data(self): + """Check that a ctypes struct/union matches its C equivalent. + + This compares with data from get_generated_test_data(), a list of: + - name (str) + - size (int) + - alignment (int) + - for each field, three snapshots of memory, as bytes: + - memory after the field is set to -1 + - memory after the field is set to 1 + - memory after the field is set to 0 + + or: + - None + - reason to skip the test (str) + + This does depend on the C compiler keeping padding bits unchanged. + Common compilers seem to do so. + """ + for name, cls in TESTCASES.items(): + with self.subTest(name=name): + self.check_struct_or_union(cls) + if _maybe_skip := getattr(cls, '_maybe_skip', None): + _maybe_skip() + expected = iter(_ctypes_test.get_generated_test_data(name)) + expected_name = next(expected) + if expected_name is None: + self.skipTest(next(expected)) + self.assertEqual(name, expected_name) + self.assertEqual(sizeof(cls), next(expected)) + with self.subTest('alignment'): + self.assertEqual(alignment(cls), next(expected)) + obj = cls() + ptr = pointer(obj) + for field in iterfields(cls): + for value in -1, 1, TEST_PATTERN, 0: + with self.subTest(field=field.full_name, value=value): + field.set_to(obj, value) + py_mem = string_at(ptr, sizeof(obj)) + c_mem = next(expected) + if py_mem != c_mem: + # Generate a helpful failure message + lines, requires = dump_ctype(cls) + m = "\n".join([str(field), 'in:', *lines]) + self.assertEqual(py_mem.hex(), c_mem.hex(), m) + + descriptor = field.descriptor + field_mem = py_mem[ + field.byte_offset + : field.byte_offset + descriptor.byte_size] + field_int = int.from_bytes(field_mem, sys.byteorder) + mask = (1 << descriptor.bit_size) - 1 + self.assertEqual( + (field_int >> descriptor.bit_offset) & mask, + value & mask) + + + +# The rest of this file is generating C code from a ctypes type. +# This is only meant for (and tested with) the known inputs in this file! + +def c_str_repr(string): + """Return a string as a C literal""" + return '"' + re.sub('([\"\'\\\\\n])', r'\\\1', string) + '"' + +def dump_simple_ctype(tp, variable_name='', semi=''): + """Get C type name or declaration of a scalar type + + variable_name: if given, declare the given variable + semi: a semicolon, and/or bitfield specification to tack on to the end + """ + length = getattr(tp, '_length_', None) + if length is not None: + return f'{dump_simple_ctype(tp._type_, variable_name)}[{length}]{semi}' + assert not issubclass(tp, (Structure, Union)) + return f'{tp._c_name}{maybe_space(variable_name)}{semi}' + + +def dump_ctype(tp, struct_or_union_tag='', variable_name='', semi=''): + """Get C type name or declaration of a ctype + + struct_or_union_tag: name of the struct or union + variable_name: if given, declare the given variable + semi: a semicolon, and/or bitfield specification to tack on to the end + """ + requires = set() + if issubclass(tp, (Structure, Union)): + attributes = [] + pushes = [] + pops = [] + pack = getattr(tp, '_pack_', None) + if pack is not None: + pushes.append(f'#pragma pack(push, {pack})') + pops.append(f'#pragma pack(pop)') + layout = getattr(tp, '_layout_', None) + if layout == 'ms': + # The 'ms_struct' attribute only works on x86 and PowerPC + requires.add( + 'defined(MS_WIN32) || (' + '(defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (' + 'defined(__GNUC__) || defined(__clang__)))' + ) + attributes.append('ms_struct') + if attributes: + a = f' GCC_ATTR({", ".join(attributes)})' + else: + a = '' + lines = [f'{struct_or_union(tp)}{a}{maybe_space(struct_or_union_tag)} ' +'{'] + for fielddesc in tp._fields_: + f_name, f_tp, f_bits = unpack_field_desc(*fielddesc) + if f_name in getattr(tp, '_anonymous_', ()): + f_name = '' + if f_bits is None: + subsemi = ';' + else: + if f_tp not in (c_int, c_uint): + # XLC can reportedly only handle int & unsigned int + # bitfields (the only types required by C spec) + requires.add('!defined(__xlc__)') + subsemi = f' :{f_bits};' + sub_lines, sub_requires = dump_ctype( + f_tp, variable_name=f_name, semi=subsemi) + requires.update(sub_requires) + for line in sub_lines: + lines.append(' ' + line) + lines.append(f'}}{maybe_space(variable_name)}{semi}') + return [*pushes, *lines, *reversed(pops)], requires + else: + return [dump_simple_ctype(tp, variable_name, semi)], requires + +def struct_or_union(cls): + if issubclass(cls, Structure): + return 'struct' + if issubclass(cls, Union): + return 'union' + raise TypeError(cls) + +def maybe_space(string): + if string: + return ' ' + string + return string + +def unpack_field_desc(f_name, f_tp, f_bits=None): + """Unpack a _fields_ entry into a (name, type, bits) triple""" + return f_name, f_tp, f_bits + +@dataclass +class FieldInfo: + """Information about a (possibly nested) struct/union field""" + name: str + tp: type + bits: int | None # number if this is a bit field + parent_type: type + parent: 'FieldInfo' #| None + descriptor: object + byte_offset: int + + @cached_property + def attr_path(self): + """Attribute names to get at the value of this field""" + if self.name in getattr(self.parent_type, '_anonymous_', ()): + selfpath = () + else: + selfpath = (self.name,) + if self.parent: + return (*self.parent.attr_path, *selfpath) + else: + return selfpath + + @cached_property + def full_name(self): + """Attribute names to get at the value of this field""" + return '.'.join(self.attr_path) + + def set_to(self, obj, new): + """Set the field on a given Structure/Union instance""" + for attr_name in self.attr_path[:-1]: + obj = getattr(obj, attr_name) + setattr(obj, self.attr_path[-1], new) + + @cached_property + def root(self): + if self.parent is None: + return self + else: + return self.parent + + def __repr__(self): + qname = f'{self.root.parent_type.__name__}.{self.full_name}' + try: + desc = self.descriptor + except AttributeError: + desc = '???' + return f'<{type(self).__name__} for {qname}: {desc}>' + +def iterfields(tp, parent=None): + """Get *leaf* fields of a structure or union, as FieldInfo""" + try: + fields = tp._fields_ + except AttributeError: + yield parent + else: + for fielddesc in fields: + f_name, f_tp, f_bits = unpack_field_desc(*fielddesc) + descriptor = getattr(tp, f_name) + byte_offset = descriptor.byte_offset + if parent: + byte_offset += parent.byte_offset + sub = FieldInfo(f_name, f_tp, f_bits, tp, parent, descriptor, byte_offset) + yield from iterfields(f_tp, sub) + + +if __name__ == '__main__': + # Dump C source to stdout + def output(string): + print(re.compile(r'^ +$', re.MULTILINE).sub('', string).lstrip('\n')) + output("/* Generated by Lib/test/test_ctypes/test_generated_structs.py */") + output(f"#define TEST_PATTERN {TEST_PATTERN}") + output(""" + // Append VALUE to the result. + #define APPEND(ITEM) { \\ + PyObject *item = ITEM; \\ + if (!item) { \\ + Py_DECREF(result); \\ + return NULL; \\ + } \\ + int rv = PyList_Append(result, item); \\ + Py_DECREF(item); \\ + if (rv < 0) { \\ + Py_DECREF(result); \\ + return NULL; \\ + } \\ + } + + // Set TARGET, and append a snapshot of `value`'s + // memory to the result. + #define SET_AND_APPEND(TYPE, TARGET, VAL) { \\ + TYPE v = VAL; \\ + TARGET = v; \\ + APPEND(PyBytes_FromStringAndSize( \\ + (char*)&value, sizeof(value))); \\ + } + + // Set a field to test values; append a snapshot of the memory + // after each of the operations. + #define TEST_FIELD(TYPE, TARGET) { \\ + SET_AND_APPEND(TYPE, TARGET, -1) \\ + SET_AND_APPEND(TYPE, TARGET, 1) \\ + SET_AND_APPEND(TYPE, TARGET, (TYPE)TEST_PATTERN) \\ + SET_AND_APPEND(TYPE, TARGET, 0) \\ + } + + #if defined(__GNUC__) || defined(__clang__) + #define GCC_ATTR(X) __attribute__((X)) + #else + #define GCC_ATTR(X) /* */ + #endif + + static PyObject * + get_generated_test_data(PyObject *self, PyObject *name) + { + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_TypeError, "need a string"); + return NULL; + } + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + """) + for name, cls in TESTCASES.items(): + output(""" + if (PyUnicode_CompareWithASCIIString(name, %s) == 0) { + """ % c_str_repr(name)) + lines, requires = dump_ctype(cls, struct_or_union_tag=name, semi=';') + if requires: + output(f""" + #if {" && ".join(f'({r})' for r in sorted(requires))} + """) + for line in lines: + output(' ' + line) + typename = f'{struct_or_union(cls)} {name}' + output(f""" + {typename} value; + memset(&value, 0, sizeof(value)); + APPEND(PyUnicode_FromString({c_str_repr(name)})); + APPEND(PyLong_FromLong(sizeof({typename}))); + APPEND(PyLong_FromLong(_Alignof({typename}))); + """.rstrip()) + for field in iterfields(cls): + f_tp = dump_simple_ctype(field.tp) + output(f"""\ + TEST_FIELD({f_tp}, value.{field.full_name}); + """.rstrip()) + if requires: + output(f""" + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + """) + output(""" + return result; + } + """) + + output(""" + Py_DECREF(result); + PyErr_Format(PyExc_ValueError, "unknown testcase %R", name); + return NULL; + } + + #undef GCC_ATTR + #undef TEST_FIELD + #undef SET_AND_APPEND + #undef APPEND + """) diff --git a/Lib/test/test_ctypes/test_incomplete.py b/Lib/test/test_ctypes/test_incomplete.py index 9f859793d88..fefdfe9102e 100644 --- a/Lib/test/test_ctypes/test_incomplete.py +++ b/Lib/test/test_ctypes/test_incomplete.py @@ -3,15 +3,20 @@ import warnings from ctypes import Structure, POINTER, pointer, c_char_p +# String-based "incomplete pointers" were implemented in ctypes 0.6.3 (2003, when +# ctypes was an external project). They made obsolete by the current +# incomplete *types* (setting `_fields_` late) in 0.9.5 (2005). +# ctypes was added to Python 2.5 (2006), without any mention in docs. -# The incomplete pointer example from the tutorial +# This tests incomplete pointer example from the old tutorial +# (https://svn.python.org/projects/ctypes/tags/release_0_6_3/ctypes/docs/tutorial.stx) class TestSetPointerType(unittest.TestCase): def tearDown(self): - # to not leak references, we must clean _pointer_type_cache - ctypes._reset_cache() + ctypes._pointer_type_cache_fallback.clear() def test_incomplete_example(self): - lpcell = POINTER("cell") + with self.assertWarns(DeprecationWarning): + lpcell = POINTER("cell") class cell(Structure): _fields_ = [("name", c_char_p), ("next", lpcell)] @@ -20,6 +25,8 @@ class cell(Structure): warnings.simplefilter('ignore', DeprecationWarning) ctypes.SetPointerType(lpcell, cell) + self.assertIs(POINTER(cell), lpcell) + c1 = cell() c1.name = b"foo" c2 = cell() @@ -37,7 +44,8 @@ class cell(Structure): self.assertEqual(result, [b"foo", b"bar"] * 4) def test_deprecation(self): - lpcell = POINTER("cell") + with self.assertWarns(DeprecationWarning): + lpcell = POINTER("cell") class cell(Structure): _fields_ = [("name", c_char_p), ("next", lpcell)] @@ -45,6 +53,7 @@ class cell(Structure): with self.assertWarns(DeprecationWarning): ctypes.SetPointerType(lpcell, cell) + self.assertIs(POINTER(cell), lpcell) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_internals.py b/Lib/test/test_ctypes/test_internals.py index 292633aaa4b..778da6573da 100644 --- a/Lib/test/test_ctypes/test_internals.py +++ b/Lib/test/test_ctypes/test_internals.py @@ -27,8 +27,6 @@ def test_ints(self): self.assertEqual(refcnt, sys.getrefcount(i)) self.assertEqual(ci._objects, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_c_char_p(self): s = "Hello, World".encode("ascii") refcnt = sys.getrefcount(s) @@ -64,8 +62,6 @@ class Y(Structure): x1.a, x2.b = 42, 93 self.assertEqual(y._objects, {"0": {}, "1": {}}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_xxx(self): class X(Structure): _fields_ = [("a", c_char_p), ("b", c_char_p)] diff --git a/Lib/test/test_ctypes/test_keeprefs.py b/Lib/test/test_ctypes/test_keeprefs.py index 5aa5b86fa45..5602460d5ff 100644 --- a/Lib/test/test_ctypes/test_keeprefs.py +++ b/Lib/test/test_ctypes/test_keeprefs.py @@ -1,6 +1,5 @@ import unittest -from ctypes import (Structure, POINTER, pointer, _pointer_type_cache, - c_char_p, c_int) +from ctypes import (Structure, POINTER, pointer, c_char_p, c_int) class SimpleTestCase(unittest.TestCase): @@ -12,8 +11,6 @@ def test_cint(self): x = c_int(99) self.assertEqual(x._objects, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ccharp(self): x = c_char_p() self.assertEqual(x._objects, None) @@ -35,8 +32,6 @@ class X(Structure): x.b = 99 self.assertEqual(x._objects, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ccharp_struct(self): class X(Structure): _fields_ = [("a", c_char_p), @@ -119,10 +114,6 @@ class RECT(Structure): r.a[0].x = 42 r.a[0].y = 99 - # to avoid leaking when tests are run several times - # clean up the types left in the cache. - del _pointer_type_cache[POINT] - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_libc.py b/Lib/test/test_ctypes/test_libc.py index 7716100b08f..df7dbc0ae26 100644 --- a/Lib/test/test_ctypes/test_libc.py +++ b/Lib/test/test_ctypes/test_libc.py @@ -1,3 +1,4 @@ +import ctypes import math import unittest from ctypes import (CDLL, CFUNCTYPE, POINTER, create_string_buffer, sizeof, @@ -21,6 +22,31 @@ def test_sqrt(self): self.assertEqual(lib.my_sqrt(4.0), 2.0) self.assertEqual(lib.my_sqrt(2.0), math.sqrt(2.0)) + @unittest.skipUnless(hasattr(ctypes, "c_double_complex"), + "requires C11 complex type and libffi >= 3.3.0") + def test_csqrt(self): + lib.my_csqrt.argtypes = ctypes.c_double_complex, + lib.my_csqrt.restype = ctypes.c_double_complex + self.assertEqual(lib.my_csqrt(4), 2+0j) + self.assertAlmostEqual(lib.my_csqrt(-1+0.01j), + 0.004999937502734214+1.0000124996093955j) + self.assertAlmostEqual(lib.my_csqrt(-1-0.01j), + 0.004999937502734214-1.0000124996093955j) + + lib.my_csqrtf.argtypes = ctypes.c_float_complex, + lib.my_csqrtf.restype = ctypes.c_float_complex + self.assertAlmostEqual(lib.my_csqrtf(-1+0.01j), + 0.004999937502734214+1.0000124996093955j) + self.assertAlmostEqual(lib.my_csqrtf(-1-0.01j), + 0.004999937502734214-1.0000124996093955j) + + lib.my_csqrtl.argtypes = ctypes.c_longdouble_complex, + lib.my_csqrtl.restype = ctypes.c_longdouble_complex + self.assertAlmostEqual(lib.my_csqrtl(-1+0.01j), + 0.004999937502734214+1.0000124996093955j) + self.assertAlmostEqual(lib.my_csqrtl(-1-0.01j), + 0.004999937502734214-1.0000124996093955j) + def test_qsort(self): comparefunc = CFUNCTYPE(c_int, POINTER(c_char), POINTER(c_char)) lib.my_qsort.argtypes = c_void_p, c_size_t, c_size_t, comparefunc diff --git a/Lib/test/test_ctypes/test_loading.py b/Lib/test/test_ctypes/test_loading.py index a4d54f676a6..3b8332fbb30 100644 --- a/Lib/test/test_ctypes/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -141,7 +141,7 @@ def test_1703286_B(self): 'test specific to Windows') def test_load_hasattr(self): # bpo-34816: shouldn't raise OSError - self.assertFalse(hasattr(ctypes.windll, 'test')) + self.assertNotHasAttr(ctypes.windll, 'test') @unittest.skipUnless(os.name == "nt", 'test specific to Windows') diff --git a/Lib/test/test_ctypes/test_memfunctions.py b/Lib/test/test_ctypes/test_memfunctions.py index 112b27ba48e..e3cb5db775e 100644 --- a/Lib/test/test_ctypes/test_memfunctions.py +++ b/Lib/test/test_ctypes/test_memfunctions.py @@ -5,7 +5,9 @@ create_string_buffer, string_at, create_unicode_buffer, wstring_at, memmove, memset, - c_char_p, c_byte, c_ubyte, c_wchar) + memoryview_at, c_void_p, + c_char_p, c_byte, c_ubyte, c_wchar, + addressof, byref) class MemFunctionsTest(unittest.TestCase): @@ -58,9 +60,6 @@ def test_cast(self): @support.refcount_test def test_string_at(self): s = string_at(b"foo bar") - # XXX The following may be wrong, depending on how Python - # manages string instances - self.assertEqual(2, sys.getrefcount(s)) self.assertTrue(s, "foo bar") self.assertEqual(string_at(b"foo bar", 7), b"foo bar") @@ -77,6 +76,62 @@ def test_wstring_at(self): self.assertEqual(wstring_at(a, 16), "Hello, World\0\0\0\0") self.assertEqual(wstring_at(a, 0), "") + def test_memoryview_at(self): + b = (c_byte * 10)() + + size = len(b) + for foreign_ptr in ( + b, + cast(b, c_void_p), + byref(b), + addressof(b), + ): + with self.subTest(foreign_ptr=type(foreign_ptr).__name__): + b[:] = b"initialval" + v = memoryview_at(foreign_ptr, size) + self.assertIsInstance(v, memoryview) + self.assertEqual(bytes(v), b"initialval") + + # test that writes to source buffer get reflected in memoryview + b[:] = b"0123456789" + self.assertEqual(bytes(v), b"0123456789") + + # test that writes to memoryview get reflected in source buffer + v[:] = b"9876543210" + self.assertEqual(bytes(b), b"9876543210") + + with self.assertRaises(ValueError): + memoryview_at(foreign_ptr, -1) + + with self.assertRaises(ValueError): + memoryview_at(foreign_ptr, sys.maxsize + 1) + + v0 = memoryview_at(foreign_ptr, 0) + self.assertEqual(bytes(v0), b'') + + def test_memoryview_at_readonly(self): + b = (c_byte * 10)() + + size = len(b) + for foreign_ptr in ( + b, + cast(b, c_void_p), + byref(b), + addressof(b), + ): + with self.subTest(foreign_ptr=type(foreign_ptr).__name__): + b[:] = b"initialval" + v = memoryview_at(foreign_ptr, size, readonly=True) + self.assertIsInstance(v, memoryview) + self.assertEqual(bytes(v), b"initialval") + + # test that writes to source buffer get reflected in memoryview + b[:] = b"0123456789" + self.assertEqual(bytes(v), b"0123456789") + + # test that writes to the memoryview are blocked + with self.assertRaises(TypeError): + v[:] = b"9876543210" if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_numbers.py b/Lib/test/test_ctypes/test_numbers.py index 29108a28ec1..c57c58eb002 100644 --- a/Lib/test/test_ctypes/test_numbers.py +++ b/Lib/test/test_ctypes/test_numbers.py @@ -1,12 +1,15 @@ import array +import ctypes import struct import sys import unittest +from itertools import combinations from operator import truth from ctypes import (byref, sizeof, alignment, c_char, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double, c_longdouble, c_bool) +from test.support.testcase import ComplexesAreIdenticalMixin def valid_ranges(*types): @@ -38,8 +41,28 @@ def valid_ranges(*types): signed_ranges = valid_ranges(*signed_types) bool_values = [True, False, 0, 1, -1, 5000, 'test', [], [1]] +class IntLike: + def __int__(self): + return 2 -class NumberTestCase(unittest.TestCase): +class IndexLike: + def __index__(self): + return 2 + +class FloatLike: + def __float__(self): + return 2.0 + +class ComplexLike: + def __complex__(self): + return 1+1j + + +INF = float("inf") +NAN = float("nan") + + +class NumberTestCase(unittest.TestCase, ComplexesAreIdenticalMixin): def test_default_init(self): # default values are set to zero @@ -86,9 +109,6 @@ def test_byref(self): def test_floats(self): # c_float and c_double can be created from # Python int and float - class FloatLike: - def __float__(self): - return 2.0 f = FloatLike() for t in float_types: self.assertEqual(t(2.0).value, 2.0) @@ -96,18 +116,34 @@ def __float__(self): self.assertEqual(t(2).value, 2.0) self.assertEqual(t(f).value, 2.0) + @unittest.skipUnless(hasattr(ctypes, "c_double_complex"), + "requires C11 complex type") + def test_complex(self): + for t in [ctypes.c_double_complex, ctypes.c_float_complex, + ctypes.c_longdouble_complex]: + self.assertEqual(t(1).value, 1+0j) + self.assertEqual(t(1.0).value, 1+0j) + self.assertEqual(t(1+0.125j).value, 1+0.125j) + self.assertEqual(t(IndexLike()).value, 2+0j) + self.assertEqual(t(FloatLike()).value, 2+0j) + self.assertEqual(t(ComplexLike()).value, 1+1j) + + @unittest.skipUnless(hasattr(ctypes, "c_double_complex"), + "requires C11 complex type") + def test_complex_round_trip(self): + # Ensure complexes transformed exactly. The CMPLX macro should + # preserve special components (like inf/nan or signed zero). + values = [complex(*_) for _ in combinations([1, -1, 0.0, -0.0, 2, + -3, INF, -INF, NAN], 2)] + for z in values: + for t in [ctypes.c_double_complex, ctypes.c_float_complex, + ctypes.c_longdouble_complex]: + with self.subTest(z=z, type=t): + self.assertComplexesAreIdentical(z, t(z).value) + def test_integers(self): - class FloatLike: - def __float__(self): - return 2.0 f = FloatLike() - class IntLike: - def __int__(self): - return 2 d = IntLike() - class IndexLike: - def __index__(self): - return 2 i = IndexLike() # integers cannot be constructed from floats, # but from integer-like objects diff --git a/Lib/test/test_ctypes/test_objects.py b/Lib/test/test_ctypes/test_objects.py index 8db1cd873fd..fb01421b955 100644 --- a/Lib/test/test_ctypes/test_objects.py +++ b/Lib/test/test_ctypes/test_objects.py @@ -58,8 +58,7 @@ def load_tests(loader, tests, pattern): - # TODO: RUSTPYTHON - doctest disabled due to null terminator in _objects - # tests.addTest(doctest.DocTestSuite()) + tests.addTest(doctest.DocTestSuite()) return tests diff --git a/Lib/test/test_ctypes/test_parameters.py b/Lib/test/test_ctypes/test_parameters.py index 1a6ddb91a7d..46f8ff93efa 100644 --- a/Lib/test/test_ctypes/test_parameters.py +++ b/Lib/test/test_ctypes/test_parameters.py @@ -4,7 +4,7 @@ from ctypes import (CDLL, PyDLL, ArgumentError, Structure, Array, Union, _Pointer, _SimpleCData, _CFuncPtr, - POINTER, pointer, byref, + POINTER, pointer, byref, sizeof, c_void_p, c_char_p, c_wchar_p, py_object, c_bool, c_char, c_wchar, @@ -88,19 +88,33 @@ def test_c_char(self): with self.assertRaises(TypeError) as cm: c_char.from_param(b"abc") self.assertEqual(str(cm.exception), - "one character bytes, bytearray or integer expected") + "one character bytes, bytearray, or an integer " + "in range(256) expected, not bytes of length 3") def test_c_wchar(self): with self.assertRaises(TypeError) as cm: c_wchar.from_param("abc") self.assertEqual(str(cm.exception), - "one character unicode string expected") + "a unicode character expected, not a string of length 3") + with self.assertRaises(TypeError) as cm: + c_wchar.from_param("") + self.assertEqual(str(cm.exception), + "a unicode character expected, not a string of length 0") with self.assertRaises(TypeError) as cm: c_wchar.from_param(123) self.assertEqual(str(cm.exception), - "unicode string expected instead of int instance") + "a unicode character expected, not instance of int") + + if sizeof(c_wchar) < 4: + with self.assertRaises(TypeError) as cm: + c_wchar.from_param('\U0001f40d') + self.assertEqual(str(cm.exception), + "the string '\\U0001f40d' cannot be converted to " + "a single wchar_t character") + + def test_int_pointers(self): LPINT = POINTER(c_int) diff --git a/Lib/test/test_ctypes/test_pep3118.py b/Lib/test/test_ctypes/test_pep3118.py index 06b2ccecade..11a0744f5a8 100644 --- a/Lib/test/test_ctypes/test_pep3118.py +++ b/Lib/test/test_ctypes/test_pep3118.py @@ -81,6 +81,7 @@ class Point(Structure): class PackedPoint(Structure): _pack_ = 2 + _layout_ = 'ms' _fields_ = [("x", c_long), ("y", c_long)] class PointMidPad(Structure): @@ -88,6 +89,7 @@ class PointMidPad(Structure): class PackedPointMidPad(Structure): _pack_ = 2 + _layout_ = 'ms' _fields_ = [("x", c_byte), ("y", c_uint64)] class PointEndPad(Structure): @@ -95,6 +97,7 @@ class PointEndPad(Structure): class PackedPointEndPad(Structure): _pack_ = 2 + _layout_ = 'ms' _fields_ = [("x", c_uint64), ("y", c_byte)] class Point2(Structure): diff --git a/Lib/test/test_ctypes/test_pickling.py b/Lib/test/test_ctypes/test_pickling.py index 9d433fc69de..8f8c09f1fb6 100644 --- a/Lib/test/test_ctypes/test_pickling.py +++ b/Lib/test/test_ctypes/test_pickling.py @@ -3,7 +3,7 @@ from ctypes import (CDLL, Structure, CFUNCTYPE, pointer, c_void_p, c_char_p, c_wchar_p, c_char, c_wchar, c_int, c_double) -from test.support import import_helper +from test.support import import_helper, thread_unsafe _ctypes_test = import_helper.import_module("_ctypes_test") @@ -21,7 +21,6 @@ def __init__(self, *args, **kw): class Y(X): _fields_ = [("str", c_char_p)] - class PickleTest: def dumps(self, item): return pickle.dumps(item, self.proto) @@ -39,6 +38,7 @@ def test_simple(self): self.assertEqual(memoryview(src).tobytes(), memoryview(dst).tobytes()) + @thread_unsafe('not thread safe') def test_struct(self): X.init_called = 0 diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index ed4541335df..771cc8fbe0e 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -1,15 +1,18 @@ import array import ctypes +import gc import sys import unittest from ctypes import (CDLL, CFUNCTYPE, Structure, - POINTER, pointer, _Pointer, _pointer_type_cache, + POINTER, pointer, _Pointer, byref, sizeof, c_void_p, c_char_p, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) +from ctypes import _pointer_type_cache, _pointer_type_cache_fallback from test.support import import_helper +from weakref import WeakSet _ctypes_test = import_helper.import_module("_ctypes_test") from ._support import (_CData, PyCPointerType, Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE) @@ -22,6 +25,9 @@ class PointersTestCase(unittest.TestCase): + def tearDown(self): + _pointer_type_cache_fallback.clear() + def test_inheritance_hierarchy(self): self.assertEqual(_Pointer.mro(), [_Pointer, _CData, object]) @@ -127,6 +133,14 @@ def test_from_address(self): addr = a.buffer_info()[0] p = POINTER(POINTER(c_int)) + def test_pointer_from_pointer(self): + p1 = POINTER(c_int) + p2 = POINTER(p1) + + self.assertIsNot(p1, p2) + self.assertIs(p1.__pointer_type__, p2) + self.assertIs(p2._type_, p1) + def test_other(self): class Table(Structure): _fields_ = [("a", c_int), @@ -141,8 +155,6 @@ class Table(Structure): pt.contents.c = 33 - del _pointer_type_cache[Table] - def test_basic(self): p = pointer(c_int(42)) # Although a pointer can be indexed, it has no length @@ -175,6 +187,7 @@ def test_bug_1467852(self): q = pointer(y) pp[0] = q # <== self.assertEqual(p[0], 6) + def test_c_void_p(self): # http://sourceforge.net/tracker/?func=detail&aid=1518190&group_id=5470&atid=105470 if sizeof(c_void_p) == 4: @@ -193,6 +206,30 @@ def test_c_void_p(self): self.assertRaises(TypeError, c_void_p, 3.14) # make sure floats are NOT accepted self.assertRaises(TypeError, c_void_p, object()) # nor other objects + def test_read_null_pointer(self): + null_ptr = POINTER(c_int)() + with self.assertRaisesRegex(ValueError, "NULL pointer access"): + null_ptr[0] + + def test_write_null_pointer(self): + null_ptr = POINTER(c_int)() + with self.assertRaisesRegex(ValueError, "NULL pointer access"): + null_ptr[0] = 1 + + def test_set_pointer_to_null_and_read(self): + class Bar(Structure): + _fields_ = [("values", POINTER(c_int))] + + bar = Bar() + bar.values = (c_int * 3)(1, 2, 3) + + values = [bar.values[0], bar.values[1], bar.values[2]] + self.assertEqual(values, [1, 2, 3]) + + bar.values = None + with self.assertRaisesRegex(ValueError, "NULL pointer access"): + bar.values[0] + def test_pointers_bool(self): # NULL pointers have a boolean False value, non-NULL pointers True. self.assertEqual(bool(POINTER(c_int)()), False) @@ -210,20 +247,230 @@ def test_pointer_type_name(self): LargeNamedType = type('T' * 2 ** 25, (Structure,), {}) self.assertTrue(POINTER(LargeNamedType)) - # to not leak references, we must clean _pointer_type_cache - del _pointer_type_cache[LargeNamedType] - def test_pointer_type_str_name(self): large_string = 'T' * 2 ** 25 - P = POINTER(large_string) + with self.assertWarns(DeprecationWarning): + P = POINTER(large_string) self.assertTrue(P) - # to not leak references, we must clean _pointer_type_cache - del _pointer_type_cache[id(P)] - def test_abstract(self): self.assertRaises(TypeError, _Pointer.set_type, 42) + def test_pointer_types_equal(self): + t1 = POINTER(c_int) + t2 = POINTER(c_int) + + self.assertIs(t1, t2) + + p1 = t1(c_int(1)) + p2 = pointer(c_int(1)) + + self.assertIsInstance(p1, t1) + self.assertIsInstance(p2, t1) + + self.assertIs(type(p1), t1) + self.assertIs(type(p2), t1) + + def test_incomplete_pointer_types_still_equal(self): + with self.assertWarns(DeprecationWarning): + t1 = POINTER("LP_C") + with self.assertWarns(DeprecationWarning): + t2 = POINTER("LP_C") + + self.assertIs(t1, t2) + + def test_incomplete_pointer_types_cannot_instantiate(self): + with self.assertWarns(DeprecationWarning): + t1 = POINTER("LP_C") + with self.assertRaisesRegex(TypeError, "has no _type_"): + t1() + + def test_pointer_set_type_twice(self): + t1 = POINTER(c_int) + self.assertIs(c_int.__pointer_type__, t1) + self.assertIs(t1._type_, c_int) + + t1.set_type(c_int) + self.assertIs(c_int.__pointer_type__, t1) + self.assertIs(t1._type_, c_int) + + def test_pointer_set_wrong_type(self): + int_ptr = POINTER(c_int) + float_ptr = POINTER(c_float) + try: + class C(c_int): + pass + + t1 = POINTER(c_int) + t2 = POINTER(c_float) + t1.set_type(c_float) + self.assertEqual(t1(c_float(1.5))[0], 1.5) + self.assertIs(t1._type_, c_float) + self.assertIs(c_int.__pointer_type__, t1) + self.assertIs(c_float.__pointer_type__, float_ptr) + + t1.set_type(C) + self.assertEqual(t1(C(123))[0].value, 123) + self.assertIs(c_int.__pointer_type__, t1) + self.assertIs(c_float.__pointer_type__, float_ptr) + finally: + POINTER(c_int).set_type(c_int) + self.assertIs(POINTER(c_int), int_ptr) + self.assertIs(POINTER(c_int)._type_, c_int) + self.assertIs(c_int.__pointer_type__, int_ptr) + + def test_pointer_not_ctypes_type(self): + with self.assertRaisesRegex(TypeError, "must have storage info"): + POINTER(int) + + with self.assertRaisesRegex(TypeError, "must have storage info"): + pointer(int) + + with self.assertRaisesRegex(TypeError, "must have storage info"): + pointer(int(1)) + + def test_pointer_set_python_type(self): + p1 = POINTER(c_int) + with self.assertRaisesRegex(TypeError, "must have storage info"): + p1.set_type(int) + + def test_pointer_type_attribute_is_none(self): + class Cls(Structure): + _fields_ = ( + ('a', c_int), + ('b', c_float), + ) + + with self.assertRaisesRegex(AttributeError, ".Cls'> has no attribute '__pointer_type__'"): + Cls.__pointer_type__ + + p = POINTER(Cls) + self.assertIs(Cls.__pointer_type__, p) + + def test_arbitrary_pointer_type_attribute(self): + class Cls(Structure): + _fields_ = ( + ('a', c_int), + ('b', c_float), + ) + + garbage = 'garbage' + + P = POINTER(Cls) + self.assertIs(Cls.__pointer_type__, P) + Cls.__pointer_type__ = garbage + self.assertIs(Cls.__pointer_type__, garbage) + self.assertIs(POINTER(Cls), garbage) + self.assertIs(P._type_, Cls) + + instance = Cls(1, 2.0) + pointer = P(instance) + self.assertEqual(pointer[0].a, 1) + self.assertEqual(pointer[0].b, 2) + + del Cls.__pointer_type__ + + NewP = POINTER(Cls) + self.assertIsNot(NewP, P) + self.assertIs(Cls.__pointer_type__, NewP) + self.assertIs(P._type_, Cls) + + def test_pointer_types_factory(self): + """Shouldn't leak""" + def factory(): + class Cls(Structure): + _fields_ = ( + ('a', c_int), + ('b', c_float), + ) + + return Cls + + ws_typ = WeakSet() + ws_ptr = WeakSet() + for _ in range(10): + typ = factory() + ptr = POINTER(typ) + + ws_typ.add(typ) + ws_ptr.add(ptr) + + typ = None + ptr = None + + gc.collect() + + self.assertEqual(len(ws_typ), 0, ws_typ) + self.assertEqual(len(ws_ptr), 0, ws_ptr) + + def test_pointer_proto_missing_argtypes_error(self): + class BadType(ctypes._Pointer): + # _type_ is intentionally missing + pass + + func = ctypes.pythonapi.Py_GetVersion + func.argtypes = (BadType,) + + with self.assertRaises(ctypes.ArgumentError): + func(object()) + +class PointerTypeCacheTestCase(unittest.TestCase): + # dummy tests to check warnings and base behavior + def tearDown(self): + _pointer_type_cache_fallback.clear() + + def test_deprecated_cache_with_not_ctypes_type(self): + class C: + pass + + with self.assertWarns(DeprecationWarning): + P = POINTER("C") + + with self.assertWarns(DeprecationWarning): + self.assertIs(_pointer_type_cache["C"], P) + + with self.assertWarns(DeprecationWarning): + _pointer_type_cache[C] = P + self.assertIs(C.__pointer_type__, P) + with self.assertWarns(DeprecationWarning): + self.assertIs(_pointer_type_cache[C], P) + + def test_deprecated_cache_with_ints(self): + with self.assertWarns(DeprecationWarning): + _pointer_type_cache[123] = 456 + + with self.assertWarns(DeprecationWarning): + self.assertEqual(_pointer_type_cache[123], 456) + + def test_deprecated_cache_with_ctypes_type(self): + class C(Structure): + _fields_ = [("a", c_int), + ("b", c_int), + ("c", c_int)] + + P1 = POINTER(C) + with self.assertWarns(DeprecationWarning): + P2 = POINTER("C") + + with self.assertWarns(DeprecationWarning): + _pointer_type_cache[C] = P2 + + self.assertIs(C.__pointer_type__, P2) + self.assertIsNot(C.__pointer_type__, P1) + + with self.assertWarns(DeprecationWarning): + self.assertIs(_pointer_type_cache[C], P2) + + with self.assertWarns(DeprecationWarning): + self.assertIs(_pointer_type_cache.get(C), P2) + + def test_get_not_registered(self): + with self.assertWarns(DeprecationWarning): + self.assertIsNone(_pointer_type_cache.get(str)) + + with self.assertWarns(DeprecationWarning): + self.assertIsNone(_pointer_type_cache.get(str, None)) + def test_repeated_set_type(self): # Regression test for gh-133290 class C(Structure): diff --git a/Lib/test/test_ctypes/test_python_api.py b/Lib/test/test_ctypes/test_python_api.py index 2e68b35f8af..a1ee8a0de1e 100644 --- a/Lib/test/test_ctypes/test_python_api.py +++ b/Lib/test/test_ctypes/test_python_api.py @@ -7,7 +7,7 @@ class PythonAPITestCase(unittest.TestCase): - # TODO: RUSTPYTHON + # TODO: RUSTPYTHON - requires pythonapi (Python C API) @unittest.expectedFailure def test_PyBytes_FromStringAndSize(self): PyBytes_FromStringAndSize = pythonapi.PyBytes_FromStringAndSize @@ -59,7 +59,7 @@ def test_PyObj_FromPtr(self): del pyobj self.assertEqual(sys.getrefcount(s), ref) - # TODO: RUSTPYTHON + # TODO: RUSTPYTHON - requires pythonapi (Python C API) @unittest.expectedFailure def test_PyOS_snprintf(self): PyOS_snprintf = pythonapi.PyOS_snprintf diff --git a/Lib/test/test_ctypes/test_random_things.py b/Lib/test/test_ctypes/test_random_things.py index 3908eca0926..73ff57d925e 100644 --- a/Lib/test/test_ctypes/test_random_things.py +++ b/Lib/test/test_ctypes/test_random_things.py @@ -51,7 +51,7 @@ def expect_unraisable(self, exc_type, exc_msg=None): if exc_msg is not None: self.assertEqual(str(cm.unraisable.exc_value), exc_msg) self.assertEqual(cm.unraisable.err_msg, - f"Exception ignored on calling ctypes " + f"Exception ignored while calling ctypes " f"callback function {callback_func!r}") self.assertIsNone(cm.unraisable.object) @@ -70,8 +70,6 @@ def test_FloatDivisionError(self): with self.expect_unraisable(ZeroDivisionError): cb(0.0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_TypeErrorDivisionError(self): cb = CFUNCTYPE(c_int, c_char_p)(callback_func) err_msg = "unsupported operand type(s) for /: 'int' and 'bytes'" diff --git a/Lib/test/test_ctypes/test_refcounts.py b/Lib/test/test_ctypes/test_refcounts.py index 9e87cfc661e..1815649ceb5 100644 --- a/Lib/test/test_ctypes/test_refcounts.py +++ b/Lib/test/test_ctypes/test_refcounts.py @@ -3,7 +3,7 @@ import sys import unittest from test import support -from test.support import import_helper +from test.support import import_helper, thread_unsafe from test.support import script_helper _ctypes_test = import_helper.import_module("_ctypes_test") @@ -13,7 +13,7 @@ dll = ctypes.CDLL(_ctypes_test.__file__) - +@thread_unsafe('not thread safe') class RefcountTestCase(unittest.TestCase): @support.refcount_test def test_1(self): @@ -24,36 +24,35 @@ def test_1(self): def callback(value): return value - self.assertEqual(sys.getrefcount(callback), 2) + orig_refcount = sys.getrefcount(callback) cb = MyCallback(callback) - self.assertGreater(sys.getrefcount(callback), 2) + self.assertGreater(sys.getrefcount(callback), orig_refcount) result = f(-10, cb) self.assertEqual(result, -18) cb = None gc.collect() - self.assertEqual(sys.getrefcount(callback), 2) + self.assertEqual(sys.getrefcount(callback), orig_refcount) @support.refcount_test def test_refcount(self): def func(*args): pass - # this is the standard refcount for func - self.assertEqual(sys.getrefcount(func), 2) + orig_refcount = sys.getrefcount(func) # the CFuncPtr instance holds at least one refcount on func: f = OtherCallback(func) - self.assertGreater(sys.getrefcount(func), 2) + self.assertGreater(sys.getrefcount(func), orig_refcount) # and may release it again del f - self.assertGreaterEqual(sys.getrefcount(func), 2) + self.assertGreaterEqual(sys.getrefcount(func), orig_refcount) # but now it must be gone gc.collect() - self.assertEqual(sys.getrefcount(func), 2) + self.assertEqual(sys.getrefcount(func), orig_refcount) class X(ctypes.Structure): _fields_ = [("a", OtherCallback)] @@ -61,29 +60,29 @@ class X(ctypes.Structure): x.a = OtherCallback(func) # the CFuncPtr instance holds at least one refcount on func: - self.assertGreater(sys.getrefcount(func), 2) + self.assertGreater(sys.getrefcount(func), orig_refcount) # and may release it again del x - self.assertGreaterEqual(sys.getrefcount(func), 2) + self.assertGreaterEqual(sys.getrefcount(func), orig_refcount) # and now it must be gone again gc.collect() - self.assertEqual(sys.getrefcount(func), 2) + self.assertEqual(sys.getrefcount(func), orig_refcount) f = OtherCallback(func) # the CFuncPtr instance holds at least one refcount on func: - self.assertGreater(sys.getrefcount(func), 2) + self.assertGreater(sys.getrefcount(func), orig_refcount) # create a cycle f.cycle = f del f gc.collect() - self.assertEqual(sys.getrefcount(func), 2) - + self.assertEqual(sys.getrefcount(func), orig_refcount) +@thread_unsafe('not thread safe') class AnotherLeak(unittest.TestCase): def test_callback(self): proto = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int) diff --git a/Lib/test/test_ctypes/test_repr.py b/Lib/test/test_ctypes/test_repr.py index e7587984a92..8c85e6cbe70 100644 --- a/Lib/test/test_ctypes/test_repr.py +++ b/Lib/test/test_ctypes/test_repr.py @@ -22,12 +22,12 @@ class ReprTest(unittest.TestCase): def test_numbers(self): for typ in subclasses: base = typ.__bases__[0] - self.assertTrue(repr(base(42)).startswith(base.__name__)) - self.assertEqual(" 8\)", + ): + CField( + name="x", + type=c_byte, + byte_size=1, + byte_offset=0, + index=0, + _internal_use=True, + bit_size=7, + bit_offset=2, + ) + # __set__ and __get__ should raise a TypeError in case their self # argument is not a ctype instance. def test___set__(self): - class MyCStruct(Structure): + class MyCStruct(self.cls): _fields_ = (("field", c_int),) self.assertRaises(TypeError, MyCStruct.field.__set__, 'wrong type self', 42) - class MyCUnion(Union): - _fields_ = (("field", c_int),) - self.assertRaises(TypeError, - MyCUnion.field.__set__, 'wrong type self', 42) - def test___get__(self): - class MyCStruct(Structure): + class MyCStruct(self.cls): _fields_ = (("field", c_int),) self.assertRaises(TypeError, MyCStruct.field.__get__, 'wrong type self', 42) - class MyCUnion(Union): - _fields_ = (("field", c_int),) - self.assertRaises(TypeError, - MyCUnion.field.__get__, 'wrong type self', 42) +class StructFieldsTestCase(unittest.TestCase, FieldsTestBase): + cls = Structure + + def test_cfield_type_flags(self): + self.assertTrue(CField.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + + def test_cfield_inheritance_hierarchy(self): + self.assertEqual(CField.mro(), [CField, object]) + +class UnionFieldsTestCase(unittest.TestCase, FieldsTestBase): + cls = Union if __name__ == "__main__": diff --git a/Lib/test/test_ctypes/test_structunion.py b/Lib/test/test_ctypes/test_structunion.py new file mode 100644 index 00000000000..5b21d48d99c --- /dev/null +++ b/Lib/test/test_ctypes/test_structunion.py @@ -0,0 +1,477 @@ +"""Common tests for ctypes.Structure and ctypes.Union""" + +import unittest +import sys +from ctypes import (Structure, Union, POINTER, sizeof, alignment, + c_char, c_byte, c_ubyte, + c_short, c_ushort, c_int, c_uint, + c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double, + c_int8, c_int16, c_int32) +from ._support import (_CData, PyCStructType, UnionType, + Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) +from struct import calcsize +import contextlib +from test.support import MS_WINDOWS + + +class StructUnionTestBase: + formats = {"c": c_char, + "b": c_byte, + "B": c_ubyte, + "h": c_short, + "H": c_ushort, + "i": c_int, + "I": c_uint, + "l": c_long, + "L": c_ulong, + "q": c_longlong, + "Q": c_ulonglong, + "f": c_float, + "d": c_double, + } + + def test_subclass(self): + class X(self.cls): + _fields_ = [("a", c_int)] + + class Y(X): + _fields_ = [("b", c_int)] + + class Z(X): + pass + + self.assertEqual(sizeof(X), sizeof(c_int)) + self.check_sizeof(Y, + struct_size=sizeof(c_int)*2, + union_size=sizeof(c_int)) + self.assertEqual(sizeof(Z), sizeof(c_int)) + self.assertEqual(X._fields_, [("a", c_int)]) + self.assertEqual(Y._fields_, [("b", c_int)]) + self.assertEqual(Z._fields_, [("a", c_int)]) + + def test_subclass_delayed(self): + class X(self.cls): + pass + self.assertEqual(sizeof(X), 0) + X._fields_ = [("a", c_int)] + + class Y(X): + pass + self.assertEqual(sizeof(Y), sizeof(X)) + Y._fields_ = [("b", c_int)] + + class Z(X): + pass + + self.assertEqual(sizeof(X), sizeof(c_int)) + self.check_sizeof(Y, + struct_size=sizeof(c_int)*2, + union_size=sizeof(c_int)) + self.assertEqual(sizeof(Z), sizeof(c_int)) + self.assertEqual(X._fields_, [("a", c_int)]) + self.assertEqual(Y._fields_, [("b", c_int)]) + self.assertEqual(Z._fields_, [("a", c_int)]) + + def test_inheritance_hierarchy(self): + self.assertEqual(self.cls.mro(), [self.cls, _CData, object]) + self.assertEqual(type(self.metacls), type) + + def test_type_flags(self): + for cls in self.cls, self.metacls: + with self.subTest(cls=cls): + self.assertTrue(cls.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(cls.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewClass = self.metacls.__new__(self.metacls, 'NewClass', + (self.cls,), {}) + for cls in self.cls, NewClass: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(self.cls): + _fields_ = [("x", c_char), + ("y", c_char)] + with self.assertRaisesRegex(SystemError, "already initialized"): + self.metacls.__init__(T, 'ptr', (), {}) + + def test_alignment(self): + class X(self.cls): + _fields_ = [("x", c_char * 3)] + self.assertEqual(alignment(X), calcsize("s")) + self.assertEqual(sizeof(X), calcsize("3s")) + + class Y(self.cls): + _fields_ = [("x", c_char * 3), + ("y", c_int)] + self.assertEqual(alignment(Y), alignment(c_int)) + self.check_sizeof(Y, + struct_size=calcsize("3s i"), + union_size=max(calcsize("3s"), calcsize("i"))) + + class SI(self.cls): + _fields_ = [("a", X), + ("b", Y)] + self.assertEqual(alignment(SI), max(alignment(Y), alignment(X))) + self.check_sizeof(SI, + struct_size=calcsize("3s0i 3si 0i"), + union_size=max(calcsize("3s"), calcsize("i"))) + + class IS(self.cls): + _fields_ = [("b", Y), + ("a", X)] + + self.assertEqual(alignment(SI), max(alignment(X), alignment(Y))) + self.check_sizeof(IS, + struct_size=calcsize("3si 3s 0i"), + union_size=max(calcsize("3s"), calcsize("i"))) + + class XX(self.cls): + _fields_ = [("a", X), + ("b", X)] + self.assertEqual(alignment(XX), alignment(X)) + self.check_sizeof(XX, + struct_size=calcsize("3s 3s 0s"), + union_size=calcsize("3s")) + + def test_empty(self): + # I had problems with these + # + # Although these are pathological cases: Empty Structures! + class X(self.cls): + _fields_ = [] + + # Is this really the correct alignment, or should it be 0? + self.assertTrue(alignment(X) == 1) + self.assertTrue(sizeof(X) == 0) + + class XX(self.cls): + _fields_ = [("a", X), + ("b", X)] + + self.assertEqual(alignment(XX), 1) + self.assertEqual(sizeof(XX), 0) + + def test_fields(self): + # test the offset and size attributes of Structure/Union fields. + class X(self.cls): + _fields_ = [("x", c_int), + ("y", c_char)] + + self.assertEqual(X.x.offset, 0) + self.assertEqual(X.x.size, sizeof(c_int)) + + if self.cls == Structure: + self.assertEqual(X.y.offset, sizeof(c_int)) + else: + self.assertEqual(X.y.offset, 0) + self.assertEqual(X.y.size, sizeof(c_char)) + + # readonly + self.assertRaises((TypeError, AttributeError), setattr, X.x, "offset", 92) + self.assertRaises((TypeError, AttributeError), setattr, X.x, "size", 92) + + # XXX Should we check nested data types also? + # offset is always relative to the class... + + def test_field_descriptor_attributes(self): + """Test information provided by the descriptors""" + class Inner(Structure): + _fields_ = [ + ("a", c_int16), + ("b", c_int8, 1), + ("c", c_int8, 2), + ] + class X(self.cls): + _fields_ = [ + ("x", c_int32), + ("y", c_int16, 1), + ("_", Inner), + ] + _anonymous_ = ["_"] + + field_names = "xy_abc" + + # name + + for name in field_names: + with self.subTest(name=name): + self.assertEqual(getattr(X, name).name, name) + + # type + + expected_types = dict( + x=c_int32, + y=c_int16, + _=Inner, + a=c_int16, + b=c_int8, + c=c_int8, + ) + assert set(expected_types) == set(field_names) + for name, tp in expected_types.items(): + with self.subTest(name=name): + self.assertEqual(getattr(X, name).type, tp) + self.assertEqual(getattr(X, name).byte_size, sizeof(tp)) + + # offset, byte_offset + + expected_offsets = dict( + x=(0, 0), + y=(0, 4), + _=(0, 6), + a=(0, 6), + b=(2, 8), + c=(2, 8), + ) + assert set(expected_offsets) == set(field_names) + for name, (union_offset, struct_offset) in expected_offsets.items(): + with self.subTest(name=name): + self.assertEqual(getattr(X, name).offset, + getattr(X, name).byte_offset) + if self.cls == Structure: + self.assertEqual(getattr(X, name).offset, struct_offset) + else: + self.assertEqual(getattr(X, name).offset, union_offset) + + # is_bitfield, bit_size, bit_offset + # size + + little_endian = (sys.byteorder == 'little') + expected_bitfield_info = dict( + # (bit_size, bit_offset) + b=(1, 0 if little_endian else 7), + c=(2, 1 if little_endian else 5), + y=(1, 0 if little_endian else 15), + ) + for name in field_names: + with self.subTest(name=name): + if info := expected_bitfield_info.get(name): + self.assertEqual(getattr(X, name).is_bitfield, True) + expected_bit_size, expected_bit_offset = info + self.assertEqual(getattr(X, name).bit_size, + expected_bit_size) + self.assertEqual(getattr(X, name).bit_offset, + expected_bit_offset) + self.assertEqual(getattr(X, name).size, + (expected_bit_size << 16) + | expected_bit_offset) + else: + self.assertEqual(getattr(X, name).is_bitfield, False) + type_size = sizeof(expected_types[name]) + self.assertEqual(getattr(X, name).bit_size, type_size * 8) + self.assertEqual(getattr(X, name).bit_offset, 0) + self.assertEqual(getattr(X, name).size, type_size) + + # is_anonymous + + for name in field_names: + with self.subTest(name=name): + self.assertEqual(getattr(X, name).is_anonymous, (name == '_')) + + + def test_invalid_field_types(self): + class POINT(self.cls): + pass + self.assertRaises(TypeError, setattr, POINT, "_fields_", [("x", 1), ("y", 2)]) + + def test_invalid_name(self): + # field name must be string + for name in b"x", 3, None: + with self.subTest(name=name): + with self.assertRaises(TypeError): + class S(self.cls): + _fields_ = [(name, c_int)] + + def test_str_name(self): + class WeirdString(str): + def __str__(self): + return "unwanted value" + class S(self.cls): + _fields_ = [(WeirdString("f"), c_int)] + self.assertEqual(S.f.name, "f") + + def test_intarray_fields(self): + class SomeInts(self.cls): + _fields_ = [("a", c_int * 4)] + + # can use tuple to initialize array (but not list!) + self.assertEqual(SomeInts((1, 2)).a[:], [1, 2, 0, 0]) + self.assertEqual(SomeInts((1, 2)).a[::], [1, 2, 0, 0]) + self.assertEqual(SomeInts((1, 2)).a[::-1], [0, 0, 2, 1]) + self.assertEqual(SomeInts((1, 2)).a[::2], [1, 0]) + self.assertEqual(SomeInts((1, 2)).a[1:5:6], [2]) + self.assertEqual(SomeInts((1, 2)).a[6:4:-1], []) + self.assertEqual(SomeInts((1, 2, 3, 4)).a[:], [1, 2, 3, 4]) + self.assertEqual(SomeInts((1, 2, 3, 4)).a[::], [1, 2, 3, 4]) + # too long + # XXX Should raise ValueError?, not RuntimeError + self.assertRaises(RuntimeError, SomeInts, (1, 2, 3, 4, 5)) + + def test_huge_field_name(self): + # issue12881: segfault with large structure field names + def create_class(length): + class S(self.cls): + _fields_ = [('x' * length, c_int)] + + for length in [10 ** i for i in range(0, 8)]: + try: + create_class(length) + except MemoryError: + # MemoryErrors are OK, we just don't want to segfault + pass + + def test_abstract_class(self): + class X(self.cls): + _abstract_ = "something" + with self.assertRaisesRegex(TypeError, r"^abstract class$"): + X() + + def test_methods(self): + self.assertIn("in_dll", dir(type(self.cls))) + self.assertIn("from_address", dir(type(self.cls))) + self.assertIn("in_dll", dir(type(self.cls))) + + def test_pack_layout_switch(self): + # Setting _pack_ implicitly sets default layout to MSVC; + # this is deprecated on non-Windows platforms. + if MS_WINDOWS: + warn_context = contextlib.nullcontext() + else: + warn_context = self.assertWarns(DeprecationWarning) + with warn_context: + class X(self.cls): + _pack_ = 1 + # _layout_ missing + _fields_ = [('a', c_int8, 1), ('b', c_int16, 2)] + + # Check MSVC layout (bitfields of different types aren't combined) + self.check_sizeof(X, struct_size=3, union_size=2) + + +class StructureTestCase(unittest.TestCase, StructUnionTestBase): + cls = Structure + metacls = PyCStructType + + def test_metaclass_name(self): + self.assertEqual(self.metacls.__name__, "PyCStructType") + + def check_sizeof(self, cls, *, struct_size, union_size): + self.assertEqual(sizeof(cls), struct_size) + + def test_simple_structs(self): + for code, tp in self.formats.items(): + class X(Structure): + _fields_ = [("x", c_char), + ("y", tp)] + self.assertEqual((sizeof(X), code), + (calcsize("c%c0%c" % (code, code)), code)) + + +class UnionTestCase(unittest.TestCase, StructUnionTestBase): + cls = Union + metacls = UnionType + + def test_metaclass_name(self): + self.assertEqual(self.metacls.__name__, "UnionType") + + def check_sizeof(self, cls, *, struct_size, union_size): + self.assertEqual(sizeof(cls), union_size) + + def test_simple_unions(self): + for code, tp in self.formats.items(): + class X(Union): + _fields_ = [("x", c_char), + ("y", tp)] + self.assertEqual((sizeof(X), code), + (calcsize("%c" % (code)), code)) + + +class PointerMemberTestBase: + def test(self): + # a Structure/Union with a POINTER field + class S(self.cls): + _fields_ = [("array", POINTER(c_int))] + + s = S() + # We can assign arrays of the correct type + s.array = (c_int * 3)(1, 2, 3) + items = [s.array[i] for i in range(3)] + self.assertEqual(items, [1, 2, 3]) + + s.array[0] = 42 + + items = [s.array[i] for i in range(3)] + self.assertEqual(items, [42, 2, 3]) + + s.array[0] = 1 + + items = [s.array[i] for i in range(3)] + self.assertEqual(items, [1, 2, 3]) + +class PointerMemberTestCase_Struct(unittest.TestCase, PointerMemberTestBase): + cls = Structure + + def test_none_to_pointer_fields(self): + class S(self.cls): + _fields_ = [("x", c_int), + ("p", POINTER(c_int))] + + s = S() + s.x = 12345678 + s.p = None + self.assertEqual(s.x, 12345678) + +class PointerMemberTestCase_Union(unittest.TestCase, PointerMemberTestBase): + cls = Union + + def test_none_to_pointer_fields(self): + class S(self.cls): + _fields_ = [("x", c_int), + ("p", POINTER(c_int))] + + s = S() + s.x = 12345678 + s.p = None + self.assertFalse(s.p) # NULL pointers are falsy + + +class TestRecursiveBase: + def test_contains_itself(self): + class Recursive(self.cls): + pass + + try: + Recursive._fields_ = [("next", Recursive)] + except AttributeError as details: + self.assertIn("Structure or union cannot contain itself", + str(details)) + else: + self.fail("Structure or union cannot contain itself") + + + def test_vice_versa(self): + class First(self.cls): + pass + class Second(self.cls): + pass + + First._fields_ = [("second", Second)] + + try: + Second._fields_ = [("first", First)] + except AttributeError as details: + self.assertIn("_fields_ is final", str(details)) + else: + self.fail("AttributeError not raised") + +class TestRecursiveStructure(unittest.TestCase, TestRecursiveBase): + cls = Structure + +class TestRecursiveUnion(unittest.TestCase, TestRecursiveBase): + cls = Union diff --git a/Lib/test/test_ctypes/test_structures.py b/Lib/test/test_ctypes/test_structures.py index 7650c80273f..92d4851d739 100644 --- a/Lib/test/test_ctypes/test_structures.py +++ b/Lib/test/test_ctypes/test_structures.py @@ -1,214 +1,32 @@ +"""Tests for ctypes.Structure + +Features common with Union should go in test_structunion.py instead. +""" + from platform import architecture as _architecture import struct import sys import unittest -from ctypes import (CDLL, Array, Structure, Union, POINTER, sizeof, byref, alignment, +from ctypes import (CDLL, Structure, Union, POINTER, sizeof, byref, c_void_p, c_char, c_wchar, c_byte, c_ubyte, - c_uint8, c_uint16, c_uint32, - c_short, c_ushort, c_int, c_uint, - c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) + c_uint8, c_uint16, c_uint32, c_int, c_uint, + c_long, c_ulong, c_longlong, c_float, c_double) from ctypes.util import find_library -from struct import calcsize from collections import namedtuple from test import support from test.support import import_helper +from ._support import StructCheckMixin _ctypes_test = import_helper.import_module("_ctypes_test") -from ._support import (_CData, PyCStructType, Py_TPFLAGS_DISALLOW_INSTANTIATION, - Py_TPFLAGS_IMMUTABLETYPE) - -class SubclassesTest(unittest.TestCase): - def test_subclass(self): - class X(Structure): - _fields_ = [("a", c_int)] - - class Y(X): - _fields_ = [("b", c_int)] - - class Z(X): - pass - - self.assertEqual(sizeof(X), sizeof(c_int)) - self.assertEqual(sizeof(Y), sizeof(c_int)*2) - self.assertEqual(sizeof(Z), sizeof(c_int)) - self.assertEqual(X._fields_, [("a", c_int)]) - self.assertEqual(Y._fields_, [("b", c_int)]) - self.assertEqual(Z._fields_, [("a", c_int)]) - - def test_subclass_delayed(self): - class X(Structure): - pass - self.assertEqual(sizeof(X), 0) - X._fields_ = [("a", c_int)] - - class Y(X): - pass - self.assertEqual(sizeof(Y), sizeof(X)) - Y._fields_ = [("b", c_int)] - - class Z(X): - pass - - self.assertEqual(sizeof(X), sizeof(c_int)) - self.assertEqual(sizeof(Y), sizeof(c_int)*2) - self.assertEqual(sizeof(Z), sizeof(c_int)) - self.assertEqual(X._fields_, [("a", c_int)]) - self.assertEqual(Y._fields_, [("b", c_int)]) - self.assertEqual(Z._fields_, [("a", c_int)]) - - -class StructureTestCase(unittest.TestCase): - formats = {"c": c_char, - "b": c_byte, - "B": c_ubyte, - "h": c_short, - "H": c_ushort, - "i": c_int, - "I": c_uint, - "l": c_long, - "L": c_ulong, - "q": c_longlong, - "Q": c_ulonglong, - "f": c_float, - "d": c_double, - } - - def test_inheritance_hierarchy(self): - self.assertEqual(Structure.mro(), [Structure, _CData, object]) - - self.assertEqual(PyCStructType.__name__, "PyCStructType") - self.assertEqual(type(PyCStructType), type) - - - def test_type_flags(self): - for cls in Structure, PyCStructType: - with self.subTest(cls=cls): - self.assertTrue(Structure.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) - self.assertFalse(Structure.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) - - def test_metaclass_details(self): - # Abstract classes (whose metaclass __init__ was not called) can't be - # instantiated directly - NewStructure = PyCStructType.__new__(PyCStructType, 'NewStructure', - (Structure,), {}) - for cls in Structure, NewStructure: - with self.subTest(cls=cls): - with self.assertRaisesRegex(TypeError, "abstract class"): - obj = cls() - - # Cannot call the metaclass __init__ more than once - class T(Structure): - _fields_ = [("x", c_char), - ("y", c_char)] - with self.assertRaisesRegex(SystemError, "already initialized"): - PyCStructType.__init__(T, 'ptr', (), {}) - - def test_simple_structs(self): - for code, tp in self.formats.items(): - class X(Structure): - _fields_ = [("x", c_char), - ("y", tp)] - self.assertEqual((sizeof(X), code), - (calcsize("c%c0%c" % (code, code)), code)) - - def test_unions(self): - for code, tp in self.formats.items(): - class X(Union): - _fields_ = [("x", c_char), - ("y", tp)] - self.assertEqual((sizeof(X), code), - (calcsize("%c" % (code)), code)) - - def test_struct_alignment(self): - class X(Structure): - _fields_ = [("x", c_char * 3)] - self.assertEqual(alignment(X), calcsize("s")) - self.assertEqual(sizeof(X), calcsize("3s")) - - class Y(Structure): - _fields_ = [("x", c_char * 3), - ("y", c_int)] - self.assertEqual(alignment(Y), alignment(c_int)) - self.assertEqual(sizeof(Y), calcsize("3si")) - - class SI(Structure): - _fields_ = [("a", X), - ("b", Y)] - self.assertEqual(alignment(SI), max(alignment(Y), alignment(X))) - self.assertEqual(sizeof(SI), calcsize("3s0i 3si 0i")) - - class IS(Structure): - _fields_ = [("b", Y), - ("a", X)] - - self.assertEqual(alignment(SI), max(alignment(X), alignment(Y))) - self.assertEqual(sizeof(IS), calcsize("3si 3s 0i")) - - class XX(Structure): - _fields_ = [("a", X), - ("b", X)] - self.assertEqual(alignment(XX), alignment(X)) - self.assertEqual(sizeof(XX), calcsize("3s 3s 0s")) - - def test_empty(self): - # I had problems with these - # - # Although these are pathological cases: Empty Structures! - class X(Structure): - _fields_ = [] - - class Y(Union): - _fields_ = [] - - # Is this really the correct alignment, or should it be 0? - self.assertTrue(alignment(X) == alignment(Y) == 1) - self.assertTrue(sizeof(X) == sizeof(Y) == 0) - - class XX(Structure): - _fields_ = [("a", X), - ("b", X)] - - self.assertEqual(alignment(XX), 1) - self.assertEqual(sizeof(XX), 0) - - def test_fields(self): - # test the offset and size attributes of Structure/Union fields. - class X(Structure): - _fields_ = [("x", c_int), - ("y", c_char)] - - self.assertEqual(X.x.offset, 0) - self.assertEqual(X.x.size, sizeof(c_int)) - - self.assertEqual(X.y.offset, sizeof(c_int)) - self.assertEqual(X.y.size, sizeof(c_char)) - - # readonly - self.assertRaises((TypeError, AttributeError), setattr, X.x, "offset", 92) - self.assertRaises((TypeError, AttributeError), setattr, X.x, "size", 92) - - class X(Union): - _fields_ = [("x", c_int), - ("y", c_char)] - - self.assertEqual(X.x.offset, 0) - self.assertEqual(X.x.size, sizeof(c_int)) - - self.assertEqual(X.y.offset, 0) - self.assertEqual(X.y.size, sizeof(c_char)) - - # readonly - self.assertRaises((TypeError, AttributeError), setattr, X.x, "offset", 92) - self.assertRaises((TypeError, AttributeError), setattr, X.x, "size", 92) - - # XXX Should we check nested data types also? - # offset is always relative to the class... +class StructureTestCase(unittest.TestCase, StructCheckMixin): def test_packed(self): class X(Structure): _fields_ = [("a", c_byte), ("b", c_longlong)] _pack_ = 1 + _layout_ = 'ms' + self.check_struct(X) self.assertEqual(sizeof(X), 9) self.assertEqual(X.b.offset, 1) @@ -217,6 +35,8 @@ class X(Structure): _fields_ = [("a", c_byte), ("b", c_longlong)] _pack_ = 2 + _layout_ = 'ms' + self.check_struct(X) self.assertEqual(sizeof(X), 10) self.assertEqual(X.b.offset, 2) @@ -227,6 +47,8 @@ class X(Structure): _fields_ = [("a", c_byte), ("b", c_longlong)] _pack_ = 4 + _layout_ = 'ms' + self.check_struct(X) self.assertEqual(sizeof(X), min(4, longlong_align) + longlong_size) self.assertEqual(X.b.offset, min(4, longlong_align)) @@ -234,26 +56,33 @@ class X(Structure): _fields_ = [("a", c_byte), ("b", c_longlong)] _pack_ = 8 + _layout_ = 'ms' + self.check_struct(X) self.assertEqual(sizeof(X), min(8, longlong_align) + longlong_size) self.assertEqual(X.b.offset, min(8, longlong_align)) - - d = {"_fields_": [("a", "b"), - ("b", "q")], - "_pack_": -1} - self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + with self.assertRaises(ValueError): + class X(Structure): + _fields_ = [("a", "b"), ("b", "q")] + _pack_ = -1 + _layout_ = "ms" @support.cpython_only def test_packed_c_limits(self): # Issue 15989 import _testcapi - d = {"_fields_": [("a", c_byte)], - "_pack_": _testcapi.INT_MAX + 1} - self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) - d = {"_fields_": [("a", c_byte)], - "_pack_": _testcapi.UINT_MAX + 2} - self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + with self.assertRaises(ValueError): + class X(Structure): + _fields_ = [("a", c_byte)] + _pack_ = _testcapi.INT_MAX + 1 + _layout_ = "ms" + + with self.assertRaises(ValueError): + class X(Structure): + _fields_ = [("a", c_byte)] + _pack_ = _testcapi.UINT_MAX + 2 + _layout_ = "ms" def test_initializers(self): class Person(Structure): @@ -274,6 +103,7 @@ class Person(Structure): def test_conflicting_initializers(self): class POINT(Structure): _fields_ = [("phi", c_float), ("rho", c_float)] + self.check_struct(POINT) # conflicting positional and keyword args self.assertRaisesRegex(TypeError, "phi", POINT, 2, 3, phi=4) self.assertRaisesRegex(TypeError, "rho", POINT, 2, 3, rho=4) @@ -284,52 +114,25 @@ class POINT(Structure): def test_keyword_initializers(self): class POINT(Structure): _fields_ = [("x", c_int), ("y", c_int)] + self.check_struct(POINT) pt = POINT(1, 2) self.assertEqual((pt.x, pt.y), (1, 2)) pt = POINT(y=2, x=1) self.assertEqual((pt.x, pt.y), (1, 2)) - def test_invalid_field_types(self): - class POINT(Structure): - pass - self.assertRaises(TypeError, setattr, POINT, "_fields_", [("x", 1), ("y", 2)]) - - def test_invalid_name(self): - # field name must be string - def declare_with_name(name): - class S(Structure): - _fields_ = [(name, c_int)] - - self.assertRaises(TypeError, declare_with_name, b"x") - - def test_intarray_fields(self): - class SomeInts(Structure): - _fields_ = [("a", c_int * 4)] - - # can use tuple to initialize array (but not list!) - self.assertEqual(SomeInts((1, 2)).a[:], [1, 2, 0, 0]) - self.assertEqual(SomeInts((1, 2)).a[::], [1, 2, 0, 0]) - self.assertEqual(SomeInts((1, 2)).a[::-1], [0, 0, 2, 1]) - self.assertEqual(SomeInts((1, 2)).a[::2], [1, 0]) - self.assertEqual(SomeInts((1, 2)).a[1:5:6], [2]) - self.assertEqual(SomeInts((1, 2)).a[6:4:-1], []) - self.assertEqual(SomeInts((1, 2, 3, 4)).a[:], [1, 2, 3, 4]) - self.assertEqual(SomeInts((1, 2, 3, 4)).a[::], [1, 2, 3, 4]) - # too long - # XXX Should raise ValueError?, not RuntimeError - self.assertRaises(RuntimeError, SomeInts, (1, 2, 3, 4, 5)) - def test_nested_initializers(self): # test initializing nested structures class Phone(Structure): _fields_ = [("areacode", c_char*6), ("number", c_char*12)] + self.check_struct(Phone) class Person(Structure): _fields_ = [("name", c_char * 12), ("phone", Phone), ("age", c_int)] + self.check_struct(Person) p = Person(b"Someone", (b"1234", b"5678"), 5) @@ -342,6 +145,7 @@ def test_structures_with_wchar(self): class PersonW(Structure): _fields_ = [("name", c_wchar * 12), ("age", c_int)] + self.check_struct(PersonW) p = PersonW("Someone \xe9") self.assertEqual(p.name, "Someone \xe9") @@ -357,11 +161,13 @@ def test_init_errors(self): class Phone(Structure): _fields_ = [("areacode", c_char*6), ("number", c_char*12)] + self.check_struct(Phone) class Person(Structure): _fields_ = [("name", c_char * 12), ("phone", Phone), ("age", c_int)] + self.check_struct(Person) cls, msg = self.get_except(Person, b"Someone", (1, 2)) self.assertEqual(cls, RuntimeError) @@ -374,47 +180,29 @@ class Person(Structure): self.assertEqual(msg, "(Phone) TypeError: too many initializers") - def test_huge_field_name(self): - # issue12881: segfault with large structure field names - def create_class(length): - class S(Structure): - _fields_ = [('x' * length, c_int)] - - for length in [10 ** i for i in range(0, 8)]: - try: - create_class(length) - except MemoryError: - # MemoryErrors are OK, we just don't want to segfault - pass - def get_except(self, func, *args): try: func(*args) except Exception as detail: return detail.__class__, str(detail) - def test_abstract_class(self): - class X(Structure): - _abstract_ = "something" - # try 'X()' - cls, msg = self.get_except(eval, "X()", locals()) - self.assertEqual((cls, msg), (TypeError, "abstract class")) - - def test_methods(self): - self.assertIn("in_dll", dir(type(Structure))) - self.assertIn("from_address", dir(type(Structure))) - self.assertIn("in_dll", dir(type(Structure))) - def test_positional_args(self): # see also http://bugs.python.org/issue5042 class W(Structure): _fields_ = [("a", c_int), ("b", c_int)] + self.check_struct(W) + class X(W): _fields_ = [("c", c_int)] + self.check_struct(X) + class Y(X): pass + self.check_struct(Y) + class Z(Y): _fields_ = [("d", c_int), ("e", c_int), ("f", c_int)] + self.check_struct(Z) z = Z(1, 2, 3, 4, 5, 6) self.assertEqual((z.a, z.b, z.c, z.d, z.e, z.f), @@ -433,6 +221,7 @@ class Test(Structure): ('second', c_ulong), ('third', c_ulong), ] + self.check_struct(Test) s = Test() s.first = 0xdeadbeef @@ -462,6 +251,7 @@ class Test(Structure): ] def __del__(self): finalizer_calls.append("called") + self.check_struct(Test) s = Test(1, 2, 3) # Test the StructUnionType_paramfunc() code path which copies the @@ -491,6 +281,7 @@ class X(Structure): ('first', c_uint), ('second', c_uint) ] + self.check_struct(X) s = X() s.first = 0xdeadbeef @@ -502,11 +293,15 @@ class X(Structure): func(s) self.assertEqual(s.first, 0xdeadbeef) self.assertEqual(s.second, 0xcafebabe) - got = X.in_dll(dll, "last_tfrsuv_arg") + dll.get_last_tfrsuv_arg.argtypes = () + dll.get_last_tfrsuv_arg.restype = X + got = dll.get_last_tfrsuv_arg() self.assertEqual(s.first, got.first) self.assertEqual(s.second, got.second) def _test_issue18060(self, Vector): + # Regression tests for gh-62260 + # The call to atan2() should succeed if the # class fields were correctly cloned in the # subclasses. Otherwise, it will segfault. @@ -577,36 +372,43 @@ class Test2(Structure): _fields_ = [ ('data', c_ubyte * 16), ] + self.check_struct(Test2) class Test3AParent(Structure): _fields_ = [ ('data', c_float * 2), ] + self.check_struct(Test3AParent) class Test3A(Test3AParent): _fields_ = [ ('more_data', c_float * 2), ] + self.check_struct(Test3A) class Test3B(Structure): _fields_ = [ ('data', c_double * 2), ] + self.check_struct(Test3B) class Test3C(Structure): _fields_ = [ ("data", c_double * 4) ] + self.check_struct(Test3C) class Test3D(Structure): _fields_ = [ ("data", c_double * 8) ] + self.check_struct(Test3D) class Test3E(Structure): _fields_ = [ ("data", c_double * 9) ] + self.check_struct(Test3E) # Tests for struct Test2 @@ -698,12 +500,15 @@ class Test3E(Structure): self.assertEqual(result.data[i], float(i+1)) def test_38368(self): + # Regression test for gh-82549 class U(Union): _fields_ = [ ('f1', c_uint8 * 16), ('f2', c_uint16 * 8), ('f3', c_uint32 * 4), ] + self.check_union(U) + u = U() u.f3[0] = 0x01234567 u.f3[1] = 0x89ABCDEF @@ -719,9 +524,9 @@ class U(Union): self.assertEqual(f2, [0x4567, 0x0123, 0xcdef, 0x89ab, 0x3210, 0x7654, 0xba98, 0xfedc]) - @unittest.skipIf(True, 'Test disabled for now - see bpo-16575/bpo-16576') + @unittest.skipIf(True, 'Test disabled for now - see gh-60779/gh-60780') def test_union_by_value(self): - # See bpo-16575 + # See gh-60779 # These should mirror the structures in Modules/_ctypes/_ctypes_test.c @@ -730,18 +535,21 @@ class Nested1(Structure): ('an_int', c_int), ('another_int', c_int), ] + self.check_struct(Nested1) class Test4(Union): _fields_ = [ ('a_long', c_long), ('a_struct', Nested1), ] + self.check_struct(Test4) class Nested2(Structure): _fields_ = [ ('an_int', c_int), ('a_union', Test4), ] + self.check_struct(Nested2) class Test5(Structure): _fields_ = [ @@ -749,6 +557,7 @@ class Test5(Structure): ('nested', Nested2), ('another_int', c_int), ] + self.check_struct(Test5) test4 = Test4() dll = CDLL(_ctypes_test.__file__) @@ -800,9 +609,9 @@ class Test5(Structure): self.assertEqual(test5.nested.an_int, 0) self.assertEqual(test5.another_int, 0) - @unittest.skipIf(True, 'Test disabled for now - see bpo-16575/bpo-16576') + @unittest.skipIf(True, 'Test disabled for now - see gh-60779/gh-60780') def test_bitfield_by_value(self): - # See bpo-16576 + # See gh-60780 # These should mirror the structures in Modules/_ctypes/_ctypes_test.c @@ -813,6 +622,7 @@ class Test6(Structure): ('C', c_int, 3), ('D', c_int, 2), ] + self.check_struct(Test6) test6 = Test6() # As these are signed int fields, all are logically -1 due to sign @@ -848,6 +658,8 @@ class Test7(Structure): ('C', c_uint, 3), ('D', c_uint, 2), ] + self.check_struct(Test7) + test7 = Test7() test7.A = 1 test7.B = 3 @@ -871,6 +683,7 @@ class Test8(Union): ('C', c_int, 3), ('D', c_int, 2), ] + self.check_union(Test8) test8 = Test8() with self.assertRaises(TypeError) as ctx: @@ -881,75 +694,29 @@ class Test8(Union): self.assertEqual(ctx.exception.args[0], 'item 1 in _argtypes_ passes ' 'a union by value, which is unsupported.') + def test_do_not_share_pointer_type_cache_via_stginfo_clone(self): + # This test case calls PyCStgInfo_clone() + # for the Mid and Vector class definitions + # and checks that pointer_type cache not shared + # between subclasses. + class Base(Structure): + _fields_ = [('y', c_double), + ('x', c_double)] + base_ptr = POINTER(Base) -class PointerMemberTestCase(unittest.TestCase): - - def test(self): - # a Structure with a POINTER field - class S(Structure): - _fields_ = [("array", POINTER(c_int))] - - s = S() - # We can assign arrays of the correct type - s.array = (c_int * 3)(1, 2, 3) - items = [s.array[i] for i in range(3)] - self.assertEqual(items, [1, 2, 3]) - - # The following are bugs, but are included here because the unittests - # also describe the current behaviour. - # - # This fails with SystemError: bad arg to internal function - # or with IndexError (with a patch I have) - - s.array[0] = 42 - - items = [s.array[i] for i in range(3)] - self.assertEqual(items, [42, 2, 3]) - - s.array[0] = 1 - - items = [s.array[i] for i in range(3)] - self.assertEqual(items, [1, 2, 3]) - - def test_none_to_pointer_fields(self): - class S(Structure): - _fields_ = [("x", c_int), - ("p", POINTER(c_int))] - - s = S() - s.x = 12345678 - s.p = None - self.assertEqual(s.x, 12345678) - - -class TestRecursiveStructure(unittest.TestCase): - def test_contains_itself(self): - class Recursive(Structure): + class Mid(Base): pass + Mid._fields_ = [] + mid_ptr = POINTER(Mid) - try: - Recursive._fields_ = [("next", Recursive)] - except AttributeError as details: - self.assertIn("Structure or union cannot contain itself", - str(details)) - else: - self.fail("Structure or union cannot contain itself") - - - def test_vice_versa(self): - class First(Structure): - pass - class Second(Structure): + class Vector(Mid): pass - First._fields_ = [("second", Second)] + vector_ptr = POINTER(Vector) - try: - Second._fields_ = [("first", First)] - except AttributeError as details: - self.assertIn("_fields_ is final", str(details)) - else: - self.fail("AttributeError not raised") + self.assertIsNot(base_ptr, mid_ptr) + self.assertIsNot(base_ptr, vector_ptr) + self.assertIsNot(mid_ptr, vector_ptr) if __name__ == '__main__': diff --git a/Lib/test/test_ctypes/test_unaligned_structures.py b/Lib/test/test_ctypes/test_unaligned_structures.py index 58a00597ef5..b5fb4c0df77 100644 --- a/Lib/test/test_ctypes/test_unaligned_structures.py +++ b/Lib/test/test_ctypes/test_unaligned_structures.py @@ -19,10 +19,12 @@ c_ushort, c_uint, c_ulong, c_ulonglong]: class X(Structure): _pack_ = 1 + _layout_ = 'ms' _fields_ = [("pad", c_byte), ("value", typ)] class Y(SwappedStructure): _pack_ = 1 + _layout_ = 'ms' _fields_ = [("pad", c_byte), ("value", typ)] structures.append(X) diff --git a/Lib/test/test_ctypes/test_values.py b/Lib/test/test_ctypes/test_values.py index e0d200e2101..18554e193be 100644 --- a/Lib/test/test_ctypes/test_values.py +++ b/Lib/test/test_ctypes/test_values.py @@ -7,9 +7,8 @@ import sys import unittest from ctypes import (Structure, CDLL, POINTER, pythonapi, - _pointer_type_cache, c_ubyte, c_char_p, c_int) -from test.support import import_helper +from test.support import import_helper, thread_unsafe class ValuesTestCase(unittest.TestCase): @@ -18,6 +17,7 @@ def setUp(self): _ctypes_test = import_helper.import_module("_ctypes_test") self.ctdll = CDLL(_ctypes_test.__file__) + @thread_unsafe("static global variables aren't thread-safe") def test_an_integer(self): # This test checks and changes an integer stored inside the # _ctypes_test dll/shared lib. @@ -39,7 +39,7 @@ def test_undefined(self): class PythonValuesTestCase(unittest.TestCase): """This test only works when python itself is a dll/shared library""" - # TODO: RUSTPYTHON + # TODO: RUSTPYTHON - requires pythonapi (Python C API) @unittest.expectedFailure def test_optimizeflag(self): # This test accesses the Py_OptimizeFlag integer, which is @@ -48,8 +48,9 @@ def test_optimizeflag(self): opt = c_int.in_dll(pythonapi, "Py_OptimizeFlag").value self.assertEqual(opt, sys.flags.optimize) - # TODO: RUSTPYTHON + # TODO: RUSTPYTHON - requires pythonapi (Python C API) @unittest.expectedFailure + @thread_unsafe('overrides frozen modules') def test_frozentable(self): # Python exports a PyImport_FrozenModules symbol. This is a # pointer to an array of struct _frozen entries. The end of the @@ -100,8 +101,6 @@ class struct_frozen(Structure): "_PyImport_FrozenBootstrap example " "in Doc/library/ctypes.rst may be out of date") - del _pointer_type_cache[struct_frozen] - def test_undefined(self): self.assertRaises(ValueError, c_int.in_dll, pythonapi, "Undefined_Symbol") diff --git a/Lib/test/test_ctypes/test_win32.py b/Lib/test/test_ctypes/test_win32.py index 4de4f0379cf..bb2fc0ca222 100644 --- a/Lib/test/test_ctypes/test_win32.py +++ b/Lib/test/test_ctypes/test_win32.py @@ -5,7 +5,6 @@ import sys import unittest from ctypes import (CDLL, Structure, POINTER, pointer, sizeof, byref, - _pointer_type_cache, c_void_p, c_char, c_int, c_long) from test import support from test.support import import_helper @@ -14,11 +13,11 @@ @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class FunctionCallTestCase(unittest.TestCase): + # TODO: RUSTPYTHON: SEH not implemented, crashes with STATUS_ACCESS_VIOLATION + @unittest.skip("TODO: RUSTPYTHON") @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC") @unittest.skipIf(sys.executable.lower().endswith('_d.exe'), "SEH not enabled in debug builds") - # TODO: RUSTPYTHON - SEH not implemented - @unittest.skipIf("RustPython" in sys.version, "SEH not implemented in RustPython") def test_SEH(self): # Disable faulthandler to prevent logging the warning: # "Windows fatal exception: access violation" @@ -67,15 +66,16 @@ def test_PARAM(self): sizeof(c_void_p)) def test_COMError(self): - from _ctypes import COMError + from ctypes import COMError if support.HAVE_DOCSTRINGS: self.assertEqual(COMError.__doc__, "Raised when a COM method call failed.") - ex = COMError(-1, "text", ("details",)) + ex = COMError(-1, "text", ("descr", "source", "helpfile", 0, "progid")) self.assertEqual(ex.hresult, -1) self.assertEqual(ex.text, "text") - self.assertEqual(ex.details, ("details",)) + self.assertEqual(ex.details, + ("descr", "source", "helpfile", 0, "progid")) self.assertEqual(COMError.mro(), [COMError, Exception, BaseException, object]) @@ -146,8 +146,8 @@ class RECT(Structure): self.assertEqual(ret.top, top.value) self.assertEqual(ret.bottom, bottom.value) - # to not leak references, we must clean _pointer_type_cache - del _pointer_type_cache[RECT] + self.assertIs(PointInRect.argtypes[0], ReturnRect.argtypes[2]) + self.assertIs(PointInRect.argtypes[0], ReturnRect.argtypes[5]) if __name__ == '__main__': diff --git a/Lib/test/test_ctypes/test_win32_com_foreign_func.py b/Lib/test/test_ctypes/test_win32_com_foreign_func.py index b12e09333bd..01db602149b 100644 --- a/Lib/test/test_ctypes/test_win32_com_foreign_func.py +++ b/Lib/test/test_ctypes/test_win32_com_foreign_func.py @@ -9,8 +9,7 @@ raise unittest.SkipTest("Windows-specific test") -from _ctypes import COMError, CopyComPointer -from ctypes import HRESULT +from ctypes import COMError, CopyComPointer, HRESULT COINIT_APARTMENTTHREADED = 0x2 @@ -158,8 +157,7 @@ class IPersist(IUnknown): self.assertEqual(0, ppst.Release()) - # TODO: RUSTPYTHON - COM iid parameter handling not implemented - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - COM iid parameter handling not implemented def test_with_paramflags_and_iid(self): class IUnknown(c_void_p): QueryInterface = proto_query_interface(None, IID_IUnknown) diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py new file mode 100644 index 00000000000..1e6f69d49e9 --- /dev/null +++ b/Lib/test/test_stable_abi_ctypes.py @@ -0,0 +1,1004 @@ +# Generated by Tools/build/stable_abi.py + +"""Test that all symbols of the Stable ABI are accessible using ctypes +""" + +import sys +import unittest +from test.support.import_helper import import_module +try: + from _testcapi import get_feature_macros +except ImportError: + raise unittest.SkipTest("requires _testcapi") + +feature_macros = get_feature_macros() + +# Stable ABI is incompatible with Py_TRACE_REFS builds due to PyObject +# layout differences. +# See https://github.com/python/cpython/issues/88299#issuecomment-1113366226 +if feature_macros['Py_TRACE_REFS']: + raise unittest.SkipTest("incompatible with Py_TRACE_REFS.") + +ctypes_test = import_module('ctypes') + +class TestStableABIAvailability(unittest.TestCase): + def test_available_symbols(self): + + for symbol_name in SYMBOL_NAMES: + with self.subTest(symbol_name): + ctypes_test.pythonapi[symbol_name] + + def test_feature_macros(self): + self.assertEqual( + set(get_feature_macros()), EXPECTED_FEATURE_MACROS) + + # The feature macros for Windows are used in creating the DLL + # definition, so they must be known on all platforms. + # If we are on Windows, we check that the hardcoded data matches + # the reality. + @unittest.skipIf(sys.platform != "win32", "Windows specific test") + def test_windows_feature_macros(self): + for name, value in WINDOWS_FEATURE_MACROS.items(): + if value != 'maybe': + with self.subTest(name): + self.assertEqual(feature_macros[name], value) + +SYMBOL_NAMES = ( + + "PyAIter_Check", + "PyArg_Parse", + "PyArg_ParseTuple", + "PyArg_ParseTupleAndKeywords", + "PyArg_UnpackTuple", + "PyArg_VaParse", + "PyArg_VaParseTupleAndKeywords", + "PyArg_ValidateKeywordArguments", + "PyBaseObject_Type", + "PyBool_FromLong", + "PyBool_Type", + "PyBuffer_FillContiguousStrides", + "PyBuffer_FillInfo", + "PyBuffer_FromContiguous", + "PyBuffer_GetPointer", + "PyBuffer_IsContiguous", + "PyBuffer_Release", + "PyBuffer_SizeFromFormat", + "PyBuffer_ToContiguous", + "PyByteArrayIter_Type", + "PyByteArray_AsString", + "PyByteArray_Concat", + "PyByteArray_FromObject", + "PyByteArray_FromStringAndSize", + "PyByteArray_Resize", + "PyByteArray_Size", + "PyByteArray_Type", + "PyBytesIter_Type", + "PyBytes_AsString", + "PyBytes_AsStringAndSize", + "PyBytes_Concat", + "PyBytes_ConcatAndDel", + "PyBytes_DecodeEscape", + "PyBytes_FromFormat", + "PyBytes_FromFormatV", + "PyBytes_FromObject", + "PyBytes_FromString", + "PyBytes_FromStringAndSize", + "PyBytes_Repr", + "PyBytes_Size", + "PyBytes_Type", + "PyCFunction_Call", + "PyCFunction_GetFlags", + "PyCFunction_GetFunction", + "PyCFunction_GetSelf", + "PyCFunction_New", + "PyCFunction_NewEx", + "PyCFunction_Type", + "PyCMethod_New", + "PyCallIter_New", + "PyCallIter_Type", + "PyCallable_Check", + "PyCapsule_GetContext", + "PyCapsule_GetDestructor", + "PyCapsule_GetName", + "PyCapsule_GetPointer", + "PyCapsule_Import", + "PyCapsule_IsValid", + "PyCapsule_New", + "PyCapsule_SetContext", + "PyCapsule_SetDestructor", + "PyCapsule_SetName", + "PyCapsule_SetPointer", + "PyCapsule_Type", + "PyClassMethodDescr_Type", + "PyCodec_BackslashReplaceErrors", + "PyCodec_Decode", + "PyCodec_Decoder", + "PyCodec_Encode", + "PyCodec_Encoder", + "PyCodec_IgnoreErrors", + "PyCodec_IncrementalDecoder", + "PyCodec_IncrementalEncoder", + "PyCodec_KnownEncoding", + "PyCodec_LookupError", + "PyCodec_NameReplaceErrors", + "PyCodec_Register", + "PyCodec_RegisterError", + "PyCodec_ReplaceErrors", + "PyCodec_StreamReader", + "PyCodec_StreamWriter", + "PyCodec_StrictErrors", + "PyCodec_Unregister", + "PyCodec_XMLCharRefReplaceErrors", + "PyComplex_FromDoubles", + "PyComplex_ImagAsDouble", + "PyComplex_RealAsDouble", + "PyComplex_Type", + "PyDescr_NewClassMethod", + "PyDescr_NewGetSet", + "PyDescr_NewMember", + "PyDescr_NewMethod", + "PyDictItems_Type", + "PyDictIterItem_Type", + "PyDictIterKey_Type", + "PyDictIterValue_Type", + "PyDictKeys_Type", + "PyDictProxy_New", + "PyDictProxy_Type", + "PyDictRevIterItem_Type", + "PyDictRevIterKey_Type", + "PyDictRevIterValue_Type", + "PyDictValues_Type", + "PyDict_Clear", + "PyDict_Contains", + "PyDict_Copy", + "PyDict_DelItem", + "PyDict_DelItemString", + "PyDict_GetItem", + "PyDict_GetItemRef", + "PyDict_GetItemString", + "PyDict_GetItemStringRef", + "PyDict_GetItemWithError", + "PyDict_Items", + "PyDict_Keys", + "PyDict_Merge", + "PyDict_MergeFromSeq2", + "PyDict_New", + "PyDict_Next", + "PyDict_SetItem", + "PyDict_SetItemString", + "PyDict_Size", + "PyDict_Type", + "PyDict_Update", + "PyDict_Values", + "PyEllipsis_Type", + "PyEnum_Type", + "PyErr_BadArgument", + "PyErr_BadInternalCall", + "PyErr_CheckSignals", + "PyErr_Clear", + "PyErr_Display", + "PyErr_DisplayException", + "PyErr_ExceptionMatches", + "PyErr_Fetch", + "PyErr_Format", + "PyErr_FormatV", + "PyErr_GetExcInfo", + "PyErr_GetHandledException", + "PyErr_GetRaisedException", + "PyErr_GivenExceptionMatches", + "PyErr_NewException", + "PyErr_NewExceptionWithDoc", + "PyErr_NoMemory", + "PyErr_NormalizeException", + "PyErr_Occurred", + "PyErr_Print", + "PyErr_PrintEx", + "PyErr_ProgramText", + "PyErr_ResourceWarning", + "PyErr_Restore", + "PyErr_SetExcInfo", + "PyErr_SetFromErrno", + "PyErr_SetFromErrnoWithFilename", + "PyErr_SetFromErrnoWithFilenameObject", + "PyErr_SetFromErrnoWithFilenameObjects", + "PyErr_SetHandledException", + "PyErr_SetImportError", + "PyErr_SetImportErrorSubclass", + "PyErr_SetInterrupt", + "PyErr_SetInterruptEx", + "PyErr_SetNone", + "PyErr_SetObject", + "PyErr_SetRaisedException", + "PyErr_SetString", + "PyErr_SyntaxLocation", + "PyErr_SyntaxLocationEx", + "PyErr_WarnEx", + "PyErr_WarnExplicit", + "PyErr_WarnFormat", + "PyErr_WriteUnraisable", + "PyEval_AcquireLock", + "PyEval_AcquireThread", + "PyEval_CallFunction", + "PyEval_CallMethod", + "PyEval_CallObjectWithKeywords", + "PyEval_EvalCode", + "PyEval_EvalCodeEx", + "PyEval_EvalFrame", + "PyEval_EvalFrameEx", + "PyEval_GetBuiltins", + "PyEval_GetFrame", + "PyEval_GetFrameBuiltins", + "PyEval_GetFrameGlobals", + "PyEval_GetFrameLocals", + "PyEval_GetFuncDesc", + "PyEval_GetFuncName", + "PyEval_GetGlobals", + "PyEval_GetLocals", + "PyEval_InitThreads", + "PyEval_ReleaseLock", + "PyEval_ReleaseThread", + "PyEval_RestoreThread", + "PyEval_SaveThread", + "PyEval_ThreadsInitialized", + "PyExc_ArithmeticError", + "PyExc_AssertionError", + "PyExc_AttributeError", + "PyExc_BaseException", + "PyExc_BaseExceptionGroup", + "PyExc_BlockingIOError", + "PyExc_BrokenPipeError", + "PyExc_BufferError", + "PyExc_BytesWarning", + "PyExc_ChildProcessError", + "PyExc_ConnectionAbortedError", + "PyExc_ConnectionError", + "PyExc_ConnectionRefusedError", + "PyExc_ConnectionResetError", + "PyExc_DeprecationWarning", + "PyExc_EOFError", + "PyExc_EncodingWarning", + "PyExc_EnvironmentError", + "PyExc_Exception", + "PyExc_FileExistsError", + "PyExc_FileNotFoundError", + "PyExc_FloatingPointError", + "PyExc_FutureWarning", + "PyExc_GeneratorExit", + "PyExc_IOError", + "PyExc_ImportError", + "PyExc_ImportWarning", + "PyExc_IndentationError", + "PyExc_IndexError", + "PyExc_InterruptedError", + "PyExc_IsADirectoryError", + "PyExc_KeyError", + "PyExc_KeyboardInterrupt", + "PyExc_LookupError", + "PyExc_MemoryError", + "PyExc_ModuleNotFoundError", + "PyExc_NameError", + "PyExc_NotADirectoryError", + "PyExc_NotImplementedError", + "PyExc_OSError", + "PyExc_OverflowError", + "PyExc_PendingDeprecationWarning", + "PyExc_PermissionError", + "PyExc_ProcessLookupError", + "PyExc_RecursionError", + "PyExc_ReferenceError", + "PyExc_ResourceWarning", + "PyExc_RuntimeError", + "PyExc_RuntimeWarning", + "PyExc_StopAsyncIteration", + "PyExc_StopIteration", + "PyExc_SyntaxError", + "PyExc_SyntaxWarning", + "PyExc_SystemError", + "PyExc_SystemExit", + "PyExc_TabError", + "PyExc_TimeoutError", + "PyExc_TypeError", + "PyExc_UnboundLocalError", + "PyExc_UnicodeDecodeError", + "PyExc_UnicodeEncodeError", + "PyExc_UnicodeError", + "PyExc_UnicodeTranslateError", + "PyExc_UnicodeWarning", + "PyExc_UserWarning", + "PyExc_ValueError", + "PyExc_Warning", + "PyExc_ZeroDivisionError", + "PyExceptionClass_Name", + "PyException_GetArgs", + "PyException_GetCause", + "PyException_GetContext", + "PyException_GetTraceback", + "PyException_SetArgs", + "PyException_SetCause", + "PyException_SetContext", + "PyException_SetTraceback", + "PyFile_FromFd", + "PyFile_GetLine", + "PyFile_WriteObject", + "PyFile_WriteString", + "PyFilter_Type", + "PyFloat_AsDouble", + "PyFloat_FromDouble", + "PyFloat_FromString", + "PyFloat_GetInfo", + "PyFloat_GetMax", + "PyFloat_GetMin", + "PyFloat_Type", + "PyFrame_GetCode", + "PyFrame_GetLineNumber", + "PyFrozenSet_New", + "PyFrozenSet_Type", + "PyGC_Collect", + "PyGC_Disable", + "PyGC_Enable", + "PyGC_IsEnabled", + "PyGILState_Ensure", + "PyGILState_GetThisThreadState", + "PyGILState_Release", + "PyGetSetDescr_Type", + "PyImport_AddModule", + "PyImport_AddModuleObject", + "PyImport_AddModuleRef", + "PyImport_AppendInittab", + "PyImport_ExecCodeModule", + "PyImport_ExecCodeModuleEx", + "PyImport_ExecCodeModuleObject", + "PyImport_ExecCodeModuleWithPathnames", + "PyImport_GetImporter", + "PyImport_GetMagicNumber", + "PyImport_GetMagicTag", + "PyImport_GetModule", + "PyImport_GetModuleDict", + "PyImport_Import", + "PyImport_ImportFrozenModule", + "PyImport_ImportFrozenModuleObject", + "PyImport_ImportModule", + "PyImport_ImportModuleLevel", + "PyImport_ImportModuleLevelObject", + "PyImport_ImportModuleNoBlock", + "PyImport_ReloadModule", + "PyIndex_Check", + "PyInterpreterState_Clear", + "PyInterpreterState_Delete", + "PyInterpreterState_Get", + "PyInterpreterState_GetDict", + "PyInterpreterState_GetID", + "PyInterpreterState_New", + "PyIter_Check", + "PyIter_Next", + "PyIter_NextItem", + "PyIter_Send", + "PyListIter_Type", + "PyListRevIter_Type", + "PyList_Append", + "PyList_AsTuple", + "PyList_GetItem", + "PyList_GetItemRef", + "PyList_GetSlice", + "PyList_Insert", + "PyList_New", + "PyList_Reverse", + "PyList_SetItem", + "PyList_SetSlice", + "PyList_Size", + "PyList_Sort", + "PyList_Type", + "PyLongRangeIter_Type", + "PyLong_AsDouble", + "PyLong_AsInt", + "PyLong_AsInt32", + "PyLong_AsInt64", + "PyLong_AsLong", + "PyLong_AsLongAndOverflow", + "PyLong_AsLongLong", + "PyLong_AsLongLongAndOverflow", + "PyLong_AsNativeBytes", + "PyLong_AsSize_t", + "PyLong_AsSsize_t", + "PyLong_AsUInt32", + "PyLong_AsUInt64", + "PyLong_AsUnsignedLong", + "PyLong_AsUnsignedLongLong", + "PyLong_AsUnsignedLongLongMask", + "PyLong_AsUnsignedLongMask", + "PyLong_AsVoidPtr", + "PyLong_FromDouble", + "PyLong_FromInt32", + "PyLong_FromInt64", + "PyLong_FromLong", + "PyLong_FromLongLong", + "PyLong_FromNativeBytes", + "PyLong_FromSize_t", + "PyLong_FromSsize_t", + "PyLong_FromString", + "PyLong_FromUInt32", + "PyLong_FromUInt64", + "PyLong_FromUnsignedLong", + "PyLong_FromUnsignedLongLong", + "PyLong_FromUnsignedNativeBytes", + "PyLong_FromVoidPtr", + "PyLong_GetInfo", + "PyLong_Type", + "PyMap_Type", + "PyMapping_Check", + "PyMapping_GetItemString", + "PyMapping_GetOptionalItem", + "PyMapping_GetOptionalItemString", + "PyMapping_HasKey", + "PyMapping_HasKeyString", + "PyMapping_HasKeyStringWithError", + "PyMapping_HasKeyWithError", + "PyMapping_Items", + "PyMapping_Keys", + "PyMapping_Length", + "PyMapping_SetItemString", + "PyMapping_Size", + "PyMapping_Values", + "PyMarshal_ReadObjectFromString", + "PyMarshal_WriteObjectToString", + "PyMem_Calloc", + "PyMem_Free", + "PyMem_Malloc", + "PyMem_RawCalloc", + "PyMem_RawFree", + "PyMem_RawMalloc", + "PyMem_RawRealloc", + "PyMem_Realloc", + "PyMemberDescr_Type", + "PyMember_GetOne", + "PyMember_SetOne", + "PyMemoryView_FromBuffer", + "PyMemoryView_FromMemory", + "PyMemoryView_FromObject", + "PyMemoryView_GetContiguous", + "PyMemoryView_Type", + "PyMethodDescr_Type", + "PyModuleDef_Init", + "PyModuleDef_Type", + "PyModule_Add", + "PyModule_AddFunctions", + "PyModule_AddIntConstant", + "PyModule_AddObject", + "PyModule_AddObjectRef", + "PyModule_AddStringConstant", + "PyModule_AddType", + "PyModule_Create2", + "PyModule_ExecDef", + "PyModule_FromDefAndSpec2", + "PyModule_GetDef", + "PyModule_GetDict", + "PyModule_GetFilename", + "PyModule_GetFilenameObject", + "PyModule_GetName", + "PyModule_GetNameObject", + "PyModule_GetState", + "PyModule_New", + "PyModule_NewObject", + "PyModule_SetDocString", + "PyModule_Type", + "PyNumber_Absolute", + "PyNumber_Add", + "PyNumber_And", + "PyNumber_AsSsize_t", + "PyNumber_Check", + "PyNumber_Divmod", + "PyNumber_Float", + "PyNumber_FloorDivide", + "PyNumber_InPlaceAdd", + "PyNumber_InPlaceAnd", + "PyNumber_InPlaceFloorDivide", + "PyNumber_InPlaceLshift", + "PyNumber_InPlaceMatrixMultiply", + "PyNumber_InPlaceMultiply", + "PyNumber_InPlaceOr", + "PyNumber_InPlacePower", + "PyNumber_InPlaceRemainder", + "PyNumber_InPlaceRshift", + "PyNumber_InPlaceSubtract", + "PyNumber_InPlaceTrueDivide", + "PyNumber_InPlaceXor", + "PyNumber_Index", + "PyNumber_Invert", + "PyNumber_Long", + "PyNumber_Lshift", + "PyNumber_MatrixMultiply", + "PyNumber_Multiply", + "PyNumber_Negative", + "PyNumber_Or", + "PyNumber_Positive", + "PyNumber_Power", + "PyNumber_Remainder", + "PyNumber_Rshift", + "PyNumber_Subtract", + "PyNumber_ToBase", + "PyNumber_TrueDivide", + "PyNumber_Xor", + "PyOS_FSPath", + "PyOS_InputHook", + "PyOS_InterruptOccurred", + "PyOS_double_to_string", + "PyOS_getsig", + "PyOS_mystricmp", + "PyOS_mystrnicmp", + "PyOS_setsig", + "PyOS_snprintf", + "PyOS_string_to_double", + "PyOS_strtol", + "PyOS_strtoul", + "PyOS_vsnprintf", + "PyObject_ASCII", + "PyObject_AsCharBuffer", + "PyObject_AsFileDescriptor", + "PyObject_AsReadBuffer", + "PyObject_AsWriteBuffer", + "PyObject_Bytes", + "PyObject_Call", + "PyObject_CallFunction", + "PyObject_CallFunctionObjArgs", + "PyObject_CallMethod", + "PyObject_CallMethodObjArgs", + "PyObject_CallNoArgs", + "PyObject_CallObject", + "PyObject_Calloc", + "PyObject_CheckBuffer", + "PyObject_CheckReadBuffer", + "PyObject_ClearWeakRefs", + "PyObject_CopyData", + "PyObject_DelAttr", + "PyObject_DelAttrString", + "PyObject_DelItem", + "PyObject_DelItemString", + "PyObject_Dir", + "PyObject_Format", + "PyObject_Free", + "PyObject_GC_Del", + "PyObject_GC_IsFinalized", + "PyObject_GC_IsTracked", + "PyObject_GC_Track", + "PyObject_GC_UnTrack", + "PyObject_GenericGetAttr", + "PyObject_GenericGetDict", + "PyObject_GenericSetAttr", + "PyObject_GenericSetDict", + "PyObject_GetAIter", + "PyObject_GetAttr", + "PyObject_GetAttrString", + "PyObject_GetBuffer", + "PyObject_GetItem", + "PyObject_GetIter", + "PyObject_GetOptionalAttr", + "PyObject_GetOptionalAttrString", + "PyObject_GetTypeData", + "PyObject_HasAttr", + "PyObject_HasAttrString", + "PyObject_HasAttrStringWithError", + "PyObject_HasAttrWithError", + "PyObject_Hash", + "PyObject_HashNotImplemented", + "PyObject_Init", + "PyObject_InitVar", + "PyObject_IsInstance", + "PyObject_IsSubclass", + "PyObject_IsTrue", + "PyObject_Length", + "PyObject_Malloc", + "PyObject_Not", + "PyObject_Realloc", + "PyObject_Repr", + "PyObject_RichCompare", + "PyObject_RichCompareBool", + "PyObject_SelfIter", + "PyObject_SetAttr", + "PyObject_SetAttrString", + "PyObject_SetItem", + "PyObject_Size", + "PyObject_Str", + "PyObject_Type", + "PyObject_Vectorcall", + "PyObject_VectorcallMethod", + "PyProperty_Type", + "PyRangeIter_Type", + "PyRange_Type", + "PyReversed_Type", + "PySeqIter_New", + "PySeqIter_Type", + "PySequence_Check", + "PySequence_Concat", + "PySequence_Contains", + "PySequence_Count", + "PySequence_DelItem", + "PySequence_DelSlice", + "PySequence_Fast", + "PySequence_GetItem", + "PySequence_GetSlice", + "PySequence_In", + "PySequence_InPlaceConcat", + "PySequence_InPlaceRepeat", + "PySequence_Index", + "PySequence_Length", + "PySequence_List", + "PySequence_Repeat", + "PySequence_SetItem", + "PySequence_SetSlice", + "PySequence_Size", + "PySequence_Tuple", + "PySetIter_Type", + "PySet_Add", + "PySet_Clear", + "PySet_Contains", + "PySet_Discard", + "PySet_New", + "PySet_Pop", + "PySet_Size", + "PySet_Type", + "PySlice_AdjustIndices", + "PySlice_GetIndices", + "PySlice_GetIndicesEx", + "PySlice_New", + "PySlice_Type", + "PySlice_Unpack", + "PyState_AddModule", + "PyState_FindModule", + "PyState_RemoveModule", + "PyStructSequence_GetItem", + "PyStructSequence_New", + "PyStructSequence_NewType", + "PyStructSequence_SetItem", + "PyStructSequence_UnnamedField", + "PySuper_Type", + "PySys_AddWarnOption", + "PySys_AddWarnOptionUnicode", + "PySys_AddXOption", + "PySys_Audit", + "PySys_AuditTuple", + "PySys_FormatStderr", + "PySys_FormatStdout", + "PySys_GetObject", + "PySys_GetXOptions", + "PySys_HasWarnOptions", + "PySys_ResetWarnOptions", + "PySys_SetArgv", + "PySys_SetArgvEx", + "PySys_SetObject", + "PySys_SetPath", + "PySys_WriteStderr", + "PySys_WriteStdout", + "PyThreadState_Clear", + "PyThreadState_Delete", + "PyThreadState_DeleteCurrent", + "PyThreadState_Get", + "PyThreadState_GetDict", + "PyThreadState_GetFrame", + "PyThreadState_GetID", + "PyThreadState_GetInterpreter", + "PyThreadState_New", + "PyThreadState_SetAsyncExc", + "PyThreadState_Swap", + "PyThread_GetInfo", + "PyThread_ReInitTLS", + "PyThread_acquire_lock", + "PyThread_acquire_lock_timed", + "PyThread_allocate_lock", + "PyThread_create_key", + "PyThread_delete_key", + "PyThread_delete_key_value", + "PyThread_exit_thread", + "PyThread_free_lock", + "PyThread_get_key_value", + "PyThread_get_stacksize", + "PyThread_get_thread_ident", + "PyThread_init_thread", + "PyThread_release_lock", + "PyThread_set_key_value", + "PyThread_set_stacksize", + "PyThread_start_new_thread", + "PyThread_tss_alloc", + "PyThread_tss_create", + "PyThread_tss_delete", + "PyThread_tss_free", + "PyThread_tss_get", + "PyThread_tss_is_created", + "PyThread_tss_set", + "PyTraceBack_Here", + "PyTraceBack_Print", + "PyTraceBack_Type", + "PyTupleIter_Type", + "PyTuple_GetItem", + "PyTuple_GetSlice", + "PyTuple_New", + "PyTuple_Pack", + "PyTuple_SetItem", + "PyTuple_Size", + "PyTuple_Type", + "PyType_ClearCache", + "PyType_Freeze", + "PyType_FromMetaclass", + "PyType_FromModuleAndSpec", + "PyType_FromSpec", + "PyType_FromSpecWithBases", + "PyType_GenericAlloc", + "PyType_GenericNew", + "PyType_GetBaseByToken", + "PyType_GetFlags", + "PyType_GetFullyQualifiedName", + "PyType_GetModule", + "PyType_GetModuleByDef", + "PyType_GetModuleName", + "PyType_GetModuleState", + "PyType_GetName", + "PyType_GetQualName", + "PyType_GetSlot", + "PyType_GetTypeDataSize", + "PyType_IsSubtype", + "PyType_Modified", + "PyType_Ready", + "PyType_Type", + "PyUnicodeDecodeError_Create", + "PyUnicodeDecodeError_GetEncoding", + "PyUnicodeDecodeError_GetEnd", + "PyUnicodeDecodeError_GetObject", + "PyUnicodeDecodeError_GetReason", + "PyUnicodeDecodeError_GetStart", + "PyUnicodeDecodeError_SetEnd", + "PyUnicodeDecodeError_SetReason", + "PyUnicodeDecodeError_SetStart", + "PyUnicodeEncodeError_GetEncoding", + "PyUnicodeEncodeError_GetEnd", + "PyUnicodeEncodeError_GetObject", + "PyUnicodeEncodeError_GetReason", + "PyUnicodeEncodeError_GetStart", + "PyUnicodeEncodeError_SetEnd", + "PyUnicodeEncodeError_SetReason", + "PyUnicodeEncodeError_SetStart", + "PyUnicodeIter_Type", + "PyUnicodeTranslateError_GetEnd", + "PyUnicodeTranslateError_GetObject", + "PyUnicodeTranslateError_GetReason", + "PyUnicodeTranslateError_GetStart", + "PyUnicodeTranslateError_SetEnd", + "PyUnicodeTranslateError_SetReason", + "PyUnicodeTranslateError_SetStart", + "PyUnicode_Append", + "PyUnicode_AppendAndDel", + "PyUnicode_AsASCIIString", + "PyUnicode_AsCharmapString", + "PyUnicode_AsDecodedObject", + "PyUnicode_AsDecodedUnicode", + "PyUnicode_AsEncodedObject", + "PyUnicode_AsEncodedString", + "PyUnicode_AsEncodedUnicode", + "PyUnicode_AsLatin1String", + "PyUnicode_AsRawUnicodeEscapeString", + "PyUnicode_AsUCS4", + "PyUnicode_AsUCS4Copy", + "PyUnicode_AsUTF16String", + "PyUnicode_AsUTF32String", + "PyUnicode_AsUTF8AndSize", + "PyUnicode_AsUTF8String", + "PyUnicode_AsUnicodeEscapeString", + "PyUnicode_AsWideChar", + "PyUnicode_AsWideCharString", + "PyUnicode_BuildEncodingMap", + "PyUnicode_Compare", + "PyUnicode_CompareWithASCIIString", + "PyUnicode_Concat", + "PyUnicode_Contains", + "PyUnicode_Count", + "PyUnicode_Decode", + "PyUnicode_DecodeASCII", + "PyUnicode_DecodeCharmap", + "PyUnicode_DecodeFSDefault", + "PyUnicode_DecodeFSDefaultAndSize", + "PyUnicode_DecodeLatin1", + "PyUnicode_DecodeLocale", + "PyUnicode_DecodeLocaleAndSize", + "PyUnicode_DecodeRawUnicodeEscape", + "PyUnicode_DecodeUTF16", + "PyUnicode_DecodeUTF16Stateful", + "PyUnicode_DecodeUTF32", + "PyUnicode_DecodeUTF32Stateful", + "PyUnicode_DecodeUTF7", + "PyUnicode_DecodeUTF7Stateful", + "PyUnicode_DecodeUTF8", + "PyUnicode_DecodeUTF8Stateful", + "PyUnicode_DecodeUnicodeEscape", + "PyUnicode_EncodeFSDefault", + "PyUnicode_EncodeLocale", + "PyUnicode_Equal", + "PyUnicode_EqualToUTF8", + "PyUnicode_EqualToUTF8AndSize", + "PyUnicode_FSConverter", + "PyUnicode_FSDecoder", + "PyUnicode_Find", + "PyUnicode_FindChar", + "PyUnicode_Format", + "PyUnicode_FromEncodedObject", + "PyUnicode_FromFormat", + "PyUnicode_FromFormatV", + "PyUnicode_FromObject", + "PyUnicode_FromOrdinal", + "PyUnicode_FromString", + "PyUnicode_FromStringAndSize", + "PyUnicode_FromWideChar", + "PyUnicode_GetDefaultEncoding", + "PyUnicode_GetLength", + "PyUnicode_GetSize", + "PyUnicode_InternFromString", + "PyUnicode_InternImmortal", + "PyUnicode_InternInPlace", + "PyUnicode_IsIdentifier", + "PyUnicode_Join", + "PyUnicode_Partition", + "PyUnicode_RPartition", + "PyUnicode_RSplit", + "PyUnicode_ReadChar", + "PyUnicode_Replace", + "PyUnicode_Resize", + "PyUnicode_RichCompare", + "PyUnicode_Split", + "PyUnicode_Splitlines", + "PyUnicode_Substring", + "PyUnicode_Tailmatch", + "PyUnicode_Translate", + "PyUnicode_Type", + "PyUnicode_WriteChar", + "PyVectorcall_Call", + "PyVectorcall_NARGS", + "PyWeakref_GetObject", + "PyWeakref_GetRef", + "PyWeakref_NewProxy", + "PyWeakref_NewRef", + "PyWrapperDescr_Type", + "PyWrapper_New", + "PyZip_Type", + "Py_AddPendingCall", + "Py_AtExit", + "Py_BuildValue", + "Py_BytesMain", + "Py_CompileString", + "Py_DecRef", + "Py_DecodeLocale", + "Py_EncodeLocale", + "Py_EndInterpreter", + "Py_EnterRecursiveCall", + "Py_Exit", + "Py_FatalError", + "Py_FileSystemDefaultEncodeErrors", + "Py_FileSystemDefaultEncoding", + "Py_Finalize", + "Py_FinalizeEx", + "Py_GenericAlias", + "Py_GenericAliasType", + "Py_GetArgcArgv", + "Py_GetBuildInfo", + "Py_GetCompiler", + "Py_GetConstant", + "Py_GetConstantBorrowed", + "Py_GetCopyright", + "Py_GetExecPrefix", + "Py_GetPath", + "Py_GetPlatform", + "Py_GetPrefix", + "Py_GetProgramFullPath", + "Py_GetProgramName", + "Py_GetPythonHome", + "Py_GetRecursionLimit", + "Py_GetVersion", + "Py_HasFileSystemDefaultEncoding", + "Py_IncRef", + "Py_Initialize", + "Py_InitializeEx", + "Py_Is", + "Py_IsFalse", + "Py_IsFinalizing", + "Py_IsInitialized", + "Py_IsNone", + "Py_IsTrue", + "Py_LeaveRecursiveCall", + "Py_Main", + "Py_MakePendingCalls", + "Py_NewInterpreter", + "Py_NewRef", + "Py_PACK_FULL_VERSION", + "Py_PACK_VERSION", + "Py_REFCNT", + "Py_ReprEnter", + "Py_ReprLeave", + "Py_SetPath", + "Py_SetProgramName", + "Py_SetPythonHome", + "Py_SetRecursionLimit", + "Py_TYPE", + "Py_UTF8Mode", + "Py_VaBuildValue", + "Py_Version", + "Py_XNewRef", + "_PyArg_ParseTupleAndKeywords_SizeT", + "_PyArg_ParseTuple_SizeT", + "_PyArg_Parse_SizeT", + "_PyArg_VaParseTupleAndKeywords_SizeT", + "_PyArg_VaParse_SizeT", + "_PyErr_BadInternalCall", + "_PyObject_CallFunction_SizeT", + "_PyObject_CallMethod_SizeT", + "_PyObject_GC_New", + "_PyObject_GC_NewVar", + "_PyObject_GC_Resize", + "_PyObject_New", + "_PyObject_NewVar", + "_PyState_AddModule", + "_PyThreadState_Init", + "_PyThreadState_Prealloc", + "_PyWeakref_CallableProxyType", + "_PyWeakref_ProxyType", + "_PyWeakref_RefType", + "_Py_BuildValue_SizeT", + "_Py_CheckRecursiveCall", + "_Py_Dealloc", + "_Py_DecRef", + "_Py_EllipsisObject", + "_Py_FalseStruct", + "_Py_IncRef", + "_Py_NoneStruct", + "_Py_NotImplementedStruct", + "_Py_SetRefcnt", + "_Py_SwappedOp", + "_Py_TrueStruct", + "_Py_VaBuildValue_SizeT", +) +if feature_macros['HAVE_FORK']: + SYMBOL_NAMES += ( + 'PyOS_AfterFork', + 'PyOS_AfterFork_Child', + 'PyOS_AfterFork_Parent', + 'PyOS_BeforeFork', + ) +if feature_macros['MS_WINDOWS']: + SYMBOL_NAMES += ( + 'PyErr_SetExcFromWindowsErr', + 'PyErr_SetExcFromWindowsErrWithFilename', + 'PyErr_SetExcFromWindowsErrWithFilenameObject', + 'PyErr_SetExcFromWindowsErrWithFilenameObjects', + 'PyErr_SetFromWindowsErr', + 'PyErr_SetFromWindowsErrWithFilename', + 'PyExc_WindowsError', + 'PyUnicode_AsMBCSString', + 'PyUnicode_DecodeCodePageStateful', + 'PyUnicode_DecodeMBCS', + 'PyUnicode_DecodeMBCSStateful', + 'PyUnicode_EncodeCodePage', + ) +if feature_macros['PY_HAVE_THREAD_NATIVE_ID']: + SYMBOL_NAMES += ( + 'PyThread_get_thread_native_id', + ) +if feature_macros['Py_REF_DEBUG']: + SYMBOL_NAMES += ( + '_Py_NegativeRefcount', + '_Py_RefTotal', + ) +if feature_macros['Py_TRACE_REFS']: + SYMBOL_NAMES += ( + ) +if feature_macros['USE_STACKCHECK']: + SYMBOL_NAMES += ( + 'PyOS_CheckStack', + ) + +EXPECTED_FEATURE_MACROS = set(['HAVE_FORK', + 'MS_WINDOWS', + 'PY_HAVE_THREAD_NATIVE_ID', + 'Py_REF_DEBUG', + 'Py_TRACE_REFS', + 'USE_STACKCHECK']) +WINDOWS_FEATURE_MACROS = {'HAVE_FORK': False, + 'MS_WINDOWS': True, + 'PY_HAVE_THREAD_NATIVE_ID': True, + 'Py_REF_DEBUG': 'maybe', + 'Py_TRACE_REFS': 'maybe', + 'USE_STACKCHECK': 'maybe'} diff --git a/crates/vm/src/buffer.rs b/crates/vm/src/buffer.rs index 6cbddb7333b..33670f1c30a 100644 --- a/crates/vm/src/buffer.rs +++ b/crates/vm/src/buffer.rs @@ -228,6 +228,11 @@ impl FormatCode { let mut arg_count = 0usize; let mut codes = vec![]; while chars.peek().is_some() { + // Skip whitespace before repeat count or format char + while let Some(b' ' | b'\t' | b'\n' | b'\r') = chars.peek() { + chars.next(); + } + // determine repeat operator: let repeat = match chars.peek() { Some(b'0'..=b'9') => { @@ -246,11 +251,6 @@ impl FormatCode { _ => 1, }; - // Skip whitespace (Python ignores whitespace in format strings) - while let Some(b' ' | b'\t' | b'\n' | b'\r') = chars.peek() { - chars.next(); - } - // determine format char: let c = match chars.next() { Some(c) => c, diff --git a/crates/vm/src/protocol/object.rs b/crates/vm/src/protocol/object.rs index d0e068e5073..d2e4b31aaea 100644 --- a/crates/vm/src/protocol/object.rs +++ b/crates/vm/src/protocol/object.rs @@ -322,7 +322,12 @@ impl PyObject { match op { PyComparisonOp::Eq => Ok(Either::B(self.is(&other))), PyComparisonOp::Ne => Ok(Either::B(!self.is(&other))), - _ => Err(vm.new_unsupported_bin_op_error(self, other, op.operator_token())), + _ => Err(vm.new_type_error(format!( + "'{}' not supported between instances of '{}' and '{}'", + op.operator_token(), + self.class().name(), + other.class().name() + ))), } } #[inline(always)] diff --git a/crates/vm/src/stdlib/ctypes.rs b/crates/vm/src/stdlib/ctypes.rs index dd6a6e39091..441a5ce37e4 100644 --- a/crates/vm/src/stdlib/ctypes.rs +++ b/crates/vm/src/stdlib/ctypes.rs @@ -70,18 +70,26 @@ impl Py { } impl PyType { - /// Check if StgInfo is already initialized - prevent double initialization + /// Check if StgInfo is already initialized. + /// Raises SystemError if already initialized. pub(crate) fn check_not_initialized(&self, vm: &VirtualMachine) -> PyResult<()> { if let Some(stg_info) = self.get_type_data::() && stg_info.initialized { return Err(vm.new_exception_msg( vm.ctx.exceptions.system_error.to_owned(), - format!("StgInfo of '{}' is already initialized.", self.name()), + format!("class \"{}\" already initialized", self.name()), )); } Ok(()) } + + /// Check if StgInfo is already initialized, returning true if so. + /// Unlike check_not_initialized, does not raise an error. + pub(crate) fn is_initialized(&self) -> bool { + self.get_type_data::() + .is_some_and(|stg_info| stg_info.initialized) + } } // Dynamic type check helpers for PyCData @@ -988,6 +996,11 @@ pub(crate) mod _ctypes { super::function::INTERNAL_CAST_ADDR } + #[pyattr] + fn _memoryview_at_addr(_vm: &VirtualMachine) -> usize { + super::function::INTERNAL_MEMORYVIEW_AT_ADDR + } + #[pyfunction] fn _cast( obj: PyObjectRef, @@ -1293,6 +1306,7 @@ pub(crate) mod _ctypes { structure::PyCStructType::make_class(ctx); union::PyCUnionType::make_class(ctx); function::PyCFuncPtrType::make_class(ctx); + function::RawMemoryBuffer::make_class(ctx); extend_module!(vm, module, { "_CData" => PyCData::make_class(ctx), diff --git a/crates/vm/src/stdlib/ctypes/array.rs b/crates/vm/src/stdlib/ctypes/array.rs index ecb9c0eccf1..843dd7d5b8c 100644 --- a/crates/vm/src/stdlib/ctypes/array.rs +++ b/crates/vm/src/stdlib/ctypes/array.rs @@ -301,6 +301,16 @@ impl Initializer for PyCArrayType { #[pyclass(flags(IMMUTABLETYPE), with(Initializer, AsNumber))] impl PyCArrayType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + #[pymethod] fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { // zelf is the array type class that from_param was called on @@ -777,8 +787,9 @@ impl PyCArray { let (ptr_val, converted) = if value.is(&vm.ctx.none) { (0usize, None) } else if let Some(bytes) = value.downcast_ref::() { - let (c, ptr) = super::base::ensure_z_null_terminated(bytes, vm); - (ptr, Some(c)) + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + zelf.0.keep_alive(index, kept_alive); + (ptr, Some(value.to_owned())) } else if let Ok(int_val) = value.try_index(vm) { (int_val.as_bigint().to_usize().unwrap_or(0), None) } else { diff --git a/crates/vm/src/stdlib/ctypes/base.rs b/crates/vm/src/stdlib/ctypes/base.rs index 0ef10ea6c73..ba2b987330a 100644 --- a/crates/vm/src/stdlib/ctypes/base.rs +++ b/crates/vm/src/stdlib/ctypes/base.rs @@ -1,9 +1,9 @@ use super::array::{WCHAR_SIZE, wchar_from_bytes, wchar_to_bytes}; -use crate::builtins::{PyBytes, PyDict, PyMemoryView, PyStr, PyType, PyTypeRef}; +use crate::builtins::{PyBytes, PyDict, PyMemoryView, PyStr, PyTuple, PyType, PyTypeRef}; use crate::class::StaticType; use crate::function::{ArgBytesLike, OptionalArg, PySetterValue}; use crate::protocol::{BufferMethods, PyBuffer}; -use crate::types::{GetDescriptor, Representable}; +use crate::types::{Constructor, GetDescriptor, Representable}; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, }; @@ -97,6 +97,9 @@ pub struct StgInfo { // FFI field types for structure/union passing (inherited from base class) pub ffi_field_types: Vec, + + // Cached pointer type (non-inheritable via descriptor) + pub pointer_type: Option, } // StgInfo is stored in type_data which requires Send + Sync. @@ -141,6 +144,7 @@ impl Default for StgInfo { paramfunc: ParamFunc::None, big_endian: cfg!(target_endian = "big"), // native endian by default ffi_field_types: Vec::new(), + pointer_type: None, } } } @@ -161,6 +165,7 @@ impl StgInfo { paramfunc: ParamFunc::None, big_endian: cfg!(target_endian = "big"), // native endian by default ffi_field_types: Vec::new(), + pointer_type: None, } } @@ -212,6 +217,7 @@ impl StgInfo { paramfunc: ParamFunc::Array, big_endian: cfg!(target_endian = "big"), // native endian by default ffi_field_types: Vec::new(), + pointer_type: None, } } @@ -293,6 +299,34 @@ impl StgInfo { } } +/// __pointer_type__ getter for ctypes metaclasses. +/// Reads from StgInfo.pointer_type (non-inheritable). +pub(super) fn pointer_type_get(zelf: &Py, vm: &VirtualMachine) -> PyResult { + zelf.stg_info_opt() + .and_then(|info| info.pointer_type.clone()) + .ok_or_else(|| { + vm.new_attribute_error(format!( + "type {} has no attribute '__pointer_type__'", + zelf.name() + )) + }) +} + +/// __pointer_type__ setter for ctypes metaclasses. +/// Writes to StgInfo.pointer_type (non-inheritable). +pub(super) fn pointer_type_set( + zelf: &Py, + value: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<()> { + if let Some(mut info) = zelf.get_type_data_mut::() { + info.pointer_type = Some(value); + Ok(()) + } else { + Err(vm.new_attribute_error(format!("cannot set __pointer_type__ on {}", zelf.name()))) + } +} + /// Get PEP3118 format string for a field type /// Returns the format string considering byte order pub(super) fn get_field_format( @@ -380,25 +414,20 @@ fn vec_to_bytes(vec: Vec) -> Vec { unsafe { Vec::from_raw_parts(ptr, len, cap) } } -/// Ensure PyBytes is null-terminated. Returns (PyBytes to keep, pointer). -/// If already contains null, returns original. Otherwise creates new with null appended. +/// Ensure PyBytes data is null-terminated. Returns (kept_alive_obj, pointer). +/// The caller must keep the returned object alive to keep the pointer valid. pub(super) fn ensure_z_null_terminated( bytes: &PyBytes, vm: &VirtualMachine, ) -> (PyObjectRef, usize) { let data = bytes.as_bytes(); - if data.contains(&0) { - // Already has null, use original - let original: PyObjectRef = vm.ctx.new_bytes(data.to_vec()).into(); - (original, data.as_ptr() as usize) - } else { - // Create new with null appended - let mut buffer = data.to_vec(); + let mut buffer = data.to_vec(); + if !buffer.ends_with(&[0]) { buffer.push(0); - let ptr = buffer.as_ptr() as usize; - let new_bytes: PyObjectRef = vm.ctx.new_bytes(buffer).into(); - (new_bytes, ptr) } + let ptr = buffer.as_ptr() as usize; + let kept_alive: PyObjectRef = vm.ctx.new_bytes(buffer).into(); + (kept_alive, ptr) } /// Convert str to null-terminated wchar_t buffer. Returns (PyBytes holder, pointer). @@ -434,6 +463,11 @@ pub struct PyCData { pub objects: PyRwLock>, /// number of references we need (b_length) pub length: AtomicCell, + /// References kept alive but not visible in _objects. + /// Used for null-terminated c_char_p buffer copies, since + /// RustPython's PyBytes lacks CPython's internal trailing null. + /// Keyed by unique_key (hierarchical) so nested fields don't collide. + pub(super) kept_refs: PyRwLock>, } impl PyCData { @@ -446,6 +480,7 @@ impl PyCData { index: AtomicCell::new(0), objects: PyRwLock::new(None), length: AtomicCell::new(stg_info.length), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } @@ -458,6 +493,7 @@ impl PyCData { index: AtomicCell::new(0), objects: PyRwLock::new(objects), length: AtomicCell::new(0), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } @@ -474,6 +510,7 @@ impl PyCData { index: AtomicCell::new(0), objects: PyRwLock::new(objects), length: AtomicCell::new(length), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } @@ -494,6 +531,7 @@ impl PyCData { index: AtomicCell::new(0), objects: PyRwLock::new(None), length: AtomicCell::new(0), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } @@ -516,6 +554,7 @@ impl PyCData { index: AtomicCell::new(idx), objects: PyRwLock::new(None), length: AtomicCell::new(length), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } @@ -542,6 +581,7 @@ impl PyCData { index: AtomicCell::new(idx), objects: PyRwLock::new(None), length: AtomicCell::new(0), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } @@ -576,6 +616,7 @@ impl PyCData { index: AtomicCell::new(0), objects: PyRwLock::new(Some(objects_dict.into())), length: AtomicCell::new(length), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } @@ -797,6 +838,22 @@ impl PyCData { Ok(()) } + /// Keep a reference alive without exposing it in _objects. + /// Walks up to root object (same as keep_ref) so the reference + /// lives as long as the owning ctypes object. + /// Uses unique_key (hierarchical) so nested fields don't collide. + pub fn keep_alive(&self, index: usize, obj: PyObjectRef) { + let key = self.unique_key(index); + if let Some(base_obj) = self.base.read().clone() { + let root = Self::find_root_object(&base_obj); + if let Some(cdata) = root.downcast_ref::() { + cdata.kept_refs.write().insert(key, obj); + return; + } + } + self.kept_refs.write().insert(key, obj); + } + /// Find the root object (one without a base) by walking up the base chain fn find_root_object(obj: &PyObject) -> PyObjectRef { // Try to get base from different ctypes types @@ -942,12 +999,68 @@ impl PyCData { return Ok(()); } + // For array fields with tuple/list input, instantiate the array type + // and unpack elements as positional args (Array_init expects *args) + if let Some(proto_type) = proto.downcast_ref::() + && let Some(stg) = proto_type.stg_info_opt() + && stg.element_type.is_some() + { + let items: Option> = + if let Some(tuple) = value.downcast_ref::() { + Some(tuple.to_vec()) + } else { + value + .downcast_ref::() + .map(|list| list.borrow_vec().to_vec()) + }; + if let Some(items) = items { + let array_obj = proto_type.as_object().call(items, vm).map_err(|e| { + // Wrap errors in RuntimeError with type name prefix + let type_name = proto_type.name().to_string(); + let exc_name = e.class().name().to_string(); + let exc_args = e.args(); + let exc_msg = exc_args + .first() + .and_then(|a| a.downcast_ref::().map(|s| s.to_string())) + .unwrap_or_default(); + vm.new_runtime_error(format!("({type_name}) {exc_name}: {exc_msg}")) + })?; + if let Some(arr) = array_obj.downcast_ref::() { + let arr_buffer = arr.0.buffer.read(); + let len = core::cmp::min(arr_buffer.len(), size); + self.write_bytes_at_offset(offset, &arr_buffer[..len]); + drop(arr_buffer); + self.keep_ref(index, array_obj, vm)?; + return Ok(()); + } + } + } + // Get field type code for special handling let field_type_code = proto .get_attr("_type_", vm) .ok() .and_then(|attr| attr.downcast_ref::().map(|s| s.to_string())); + // c_char_p (z type) with bytes: store original in _objects, keep + // null-terminated copy alive separately for the pointer. + if field_type_code.as_deref() == Some("z") + && let Some(bytes_val) = value.downcast_ref::() + { + let (kept_alive, ptr) = ensure_z_null_terminated(bytes_val, vm); + let mut result = vec![0u8; size]; + let addr_bytes = ptr.to_ne_bytes(); + let len = core::cmp::min(addr_bytes.len(), size); + result[..len].copy_from_slice(&addr_bytes[..len]); + if needs_swap { + result.reverse(); + } + self.write_bytes_at_offset(offset, &result); + self.keep_ref(index, value, vm)?; + self.keep_alive(index, kept_alive); + return Ok(()); + } + let (mut bytes, converted_value) = if let Some(type_code) = &field_type_code { PyCField::value_to_bytes_for_type(type_code, &value, size, vm)? } else { @@ -1111,7 +1224,7 @@ impl PyCData { // CDataType_methods - shared across all ctypes types #[pyclassmethod] - fn from_buffer( + pub(super) fn from_buffer( cls: PyTypeRef, source: PyObjectRef, offset: OptionalArg, @@ -1122,7 +1235,7 @@ impl PyCData { } #[pyclassmethod] - fn from_buffer_copy( + pub(super) fn from_buffer_copy( cls: PyTypeRef, source: ArgBytesLike, offset: OptionalArg, @@ -1134,7 +1247,7 @@ impl PyCData { } #[pyclassmethod] - fn from_address(cls: PyTypeRef, address: isize, vm: &VirtualMachine) -> PyResult { + pub(super) fn from_address(cls: PyTypeRef, address: isize, vm: &VirtualMachine) -> PyResult { let size = { let stg_info = cls.stg_info(vm)?; stg_info.size @@ -1150,7 +1263,7 @@ impl PyCData { } #[pyclassmethod] - fn in_dll( + pub(super) fn in_dll( cls: PyTypeRef, dll: PyObjectRef, name: crate::builtins::PyStrRef, @@ -1217,78 +1330,84 @@ impl PyCData { #[pyclass(name = "CField", module = "_ctypes")] #[derive(Debug, PyPayload)] pub struct PyCField { + /// Field name + pub(crate) name: String, /// Byte offset of the field within the structure/union pub(crate) offset: isize, - /// Encoded size: for bitfields (bit_size << 16) | bit_offset, otherwise byte size - pub(crate) size: isize, + /// Byte size of the underlying type + pub(crate) byte_size_val: isize, /// Index into PyCData's object array pub(crate) index: usize, /// The ctypes type for this field pub(crate) proto: PyTypeRef, /// Flag indicating if the field is anonymous (MakeAnonFields sets this) pub(crate) anonymous: bool, -} - -#[inline(always)] -const fn num_bits(size: isize) -> isize { - size >> 16 -} - -#[inline(always)] -const fn field_size(size: isize) -> isize { - size & 0xFFFF -} - -#[inline(always)] -const fn is_bitfield(size: isize) -> bool { - (size >> 16) != 0 + /// Bitfield size in bits (0 for non-bitfield) + pub(crate) bitfield_size: u16, + /// Bit offset within the storage unit (only meaningful for bitfields) + pub(crate) bit_offset_val: u16, } impl PyCField { /// Create a new CField descriptor (non-bitfield) - pub fn new(proto: PyTypeRef, offset: isize, size: isize, index: usize) -> Self { + pub fn new( + name: String, + proto: PyTypeRef, + offset: isize, + byte_size: isize, + index: usize, + ) -> Self { Self { + name, offset, - size, + byte_size_val: byte_size, index, proto, anonymous: false, + bitfield_size: 0, + bit_offset_val: 0, } } /// Create a new CField descriptor for a bitfield - #[allow(dead_code)] pub fn new_bitfield( + name: String, proto: PyTypeRef, offset: isize, - bit_size: u16, + byte_size: isize, + bitfield_size: u16, bit_offset: u16, index: usize, ) -> Self { - let encoded_size = ((bit_size as isize) << 16) | (bit_offset as isize); Self { + name, offset, - size: encoded_size, + byte_size_val: byte_size, index, proto, anonymous: false, + bitfield_size, + bit_offset_val: bit_offset, } } - /// Get the actual byte size (for non-bitfields) or bit storage size (for bitfields) - pub fn byte_size(&self) -> usize { - field_size(self.size) as usize + /// Get the byte size of the field's underlying type + pub fn get_byte_size(&self) -> usize { + self.byte_size_val as usize } /// Create a new CField from an existing field with adjusted offset and index /// Used by MakeFields to promote anonymous fields pub fn new_from_field(fdescr: &PyCField, index_offset: usize, offset_delta: isize) -> Self { Self { + name: fdescr.name.clone(), offset: fdescr.offset + offset_delta, - size: fdescr.size, + byte_size_val: fdescr.byte_size_val, index: fdescr.index + index_offset, proto: fdescr.proto.clone(), anonymous: false, // promoted fields are not anonymous themselves + bitfield_size: fdescr.bitfield_size, + bit_offset_val: fdescr.bit_offset_val, } } @@ -1298,6 +1417,107 @@ impl PyCField { } } +impl Constructor for PyCField { + type Args = crate::function::FuncArgs; + + fn py_new(_cls: &Py, args: Self::Args, vm: &VirtualMachine) -> PyResult { + // PyCField_new_impl: requires _internal_use=True + let internal_use = if let Some(v) = args.kwargs.get("_internal_use") { + v.clone().try_to_bool(vm)? + } else { + false + }; + + if !internal_use { + return Err(vm.new_type_error( + "CField is not intended to be used directly; use it via Structure or Union fields" + .to_string(), + )); + } + + let name: String = args + .kwargs + .get("name") + .ok_or_else(|| vm.new_type_error("missing required argument: 'name'"))? + .try_to_value(vm)?; + + let field_type: PyTypeRef = args + .kwargs + .get("type") + .ok_or_else(|| vm.new_type_error("missing required argument: 'type'"))? + .clone() + .downcast() + .map_err(|_| vm.new_type_error("'type' must be a ctypes type"))?; + + let byte_size: isize = args + .kwargs + .get("byte_size") + .ok_or_else(|| vm.new_type_error("missing required argument: 'byte_size'"))? + .try_to_value(vm)?; + + let byte_offset: isize = args + .kwargs + .get("byte_offset") + .ok_or_else(|| vm.new_type_error("missing required argument: 'byte_offset'"))? + .try_to_value(vm)?; + + let index: usize = args + .kwargs + .get("index") + .ok_or_else(|| vm.new_type_error("missing required argument: 'index'"))? + .try_to_value(vm)?; + + // Validate byte_size matches the type + let type_size = super::base::get_field_size(field_type.as_object(), vm)? as isize; + if byte_size != type_size { + return Err(vm.new_value_error(format!( + "byte_size {} does not match type size {}", + byte_size, type_size + ))); + } + + let bit_size_val: Option = args + .kwargs + .get("bit_size") + .map(|v| v.try_to_value(vm)) + .transpose()?; + + let bit_offset_val: Option = args + .kwargs + .get("bit_offset") + .map(|v| v.try_to_value(vm)) + .transpose()?; + + if let Some(bs) = bit_size_val { + if bs < 0 { + return Err(vm.new_value_error("number of bits invalid for bit field".to_string())); + } + let bo = bit_offset_val.unwrap_or(0); + if bo < 0 { + return Err(vm.new_value_error("bit_offset must be >= 0".to_string())); + } + let type_bits = byte_size * 8; + if bo + bs > type_bits { + return Err(vm.new_value_error(format!( + "bit field '{}' overflows its type ({} + {} > {})", + name, bo, bs, type_bits + ))); + } + Ok(Self::new_bitfield( + name, + field_type, + byte_offset, + byte_size, + bs as u16, + bo as u16, + index, + )) + } else { + Ok(Self::new(name, field_type, byte_offset, byte_size, index)) + } + } +} + impl Representable for PyCField { fn repr_str(zelf: &Py, _vm: &VirtualMachine) -> PyResult { // Get type name from proto (which is always PyTypeRef) @@ -1305,17 +1525,15 @@ impl Representable for PyCField { // Bitfield: // Regular: - if is_bitfield(zelf.size) { - let bit_offset = field_size(zelf.size); - let bits = num_bits(zelf.size); + if zelf.bitfield_size > 0 { Ok(format!( "", - tp_name, zelf.offset, bit_offset, bits + tp_name, zelf.offset, zelf.bit_offset_val, zelf.bitfield_size )) } else { Ok(format!( "", - tp_name, zelf.offset, zelf.size + tp_name, zelf.offset, zelf.byte_size_val )) } } @@ -1340,7 +1558,7 @@ impl GetDescriptor for PyCField { }; let offset = zelf.offset as usize; - let size = zelf.byte_size(); + let size = zelf.get_byte_size(); // Get PyCData from obj (works for both Structure and Union) let cdata = PyCField::get_cdata_from_obj(&obj, vm)?; @@ -1468,15 +1686,8 @@ impl PyCField { Ok((f.to_ne_bytes().to_vec(), None)) } "z" => { - // c_char_p: store pointer to null-terminated bytes - if let Some(bytes) = value.downcast_ref::() { - let (converted, ptr) = ensure_z_null_terminated(bytes, vm); - let mut result = vec![0u8; size]; - let addr_bytes = ptr.to_ne_bytes(); - let len = core::cmp::min(addr_bytes.len(), size); - result[..len].copy_from_slice(&addr_bytes[..len]); - return Ok((result, Some(converted))); - } + // c_char_p with bytes is handled in set_field before this call. + // This handles integer address and None cases. // Integer address if let Ok(int_val) = value.try_index(vm) { let v = int_val.as_bigint().to_usize().unwrap_or(0); @@ -1583,10 +1794,7 @@ impl PyCField { } } -#[pyclass( - flags(DISALLOW_INSTANTIATION, IMMUTABLETYPE), - with(Representable, GetDescriptor) -)] +#[pyclass(flags(IMMUTABLETYPE), with(Representable, GetDescriptor, Constructor))] impl PyCField { /// Get PyCData from object (works for both Structure and Union) fn get_cdata_from_obj<'a>(obj: &'a PyObjectRef, vm: &VirtualMachine) -> PyResult<&'a PyCData> { @@ -1615,7 +1823,7 @@ impl PyCField { .ok_or_else(|| vm.new_type_error("expected CField"))?; let offset = zelf.offset as usize; - let size = zelf.byte_size(); + let size = zelf.get_byte_size(); // Get PyCData from obj (works for both Structure and Union) let cdata = Self::get_cdata_from_obj(&obj, vm)?; @@ -1650,14 +1858,65 @@ impl PyCField { } } + #[pygetset] + fn name(&self) -> String { + self.name.clone() + } + + #[pygetset(name = "type")] + fn type_(&self) -> PyTypeRef { + self.proto.clone() + } + #[pygetset] fn offset(&self) -> isize { self.offset } + #[pygetset] + fn byte_offset(&self) -> isize { + self.offset + } + #[pygetset] fn size(&self) -> isize { - self.size + // Legacy: encode as (bitfield_size << 16) | bit_offset for bitfields + if self.bitfield_size > 0 { + ((self.bitfield_size as isize) << 16) | (self.bit_offset_val as isize) + } else { + self.byte_size_val + } + } + + #[pygetset] + fn byte_size(&self) -> isize { + self.byte_size_val + } + + #[pygetset] + fn bit_offset(&self) -> isize { + self.bit_offset_val as isize + } + + #[pygetset] + fn bit_size(&self, vm: &VirtualMachine) -> PyObjectRef { + if self.bitfield_size > 0 { + vm.ctx.new_int(self.bitfield_size).into() + } else { + // Non-bitfield: bit_size = byte_size * 8 + let byte_size = self.byte_size_val as i128; + vm.ctx.new_int(byte_size * 8).into() + } + } + + #[pygetset] + fn is_bitfield(&self) -> bool { + self.bitfield_size > 0 + } + + #[pygetset] + fn is_anonymous(&self) -> bool { + self.anonymous } } @@ -2139,7 +2398,12 @@ pub(super) fn set_or_init_stginfo(type_ref: &PyType, stg_info: StgInfo) { if type_ref.init_type_data(stg_info.clone()).is_err() && let Some(mut existing) = type_ref.get_type_data_mut::() { + // Preserve pointer_type cache across StgInfo replacement + let old_pointer_type = existing.pointer_type.take(); *existing = stg_info; + if existing.pointer_type.is_none() { + existing.pointer_type = old_pointer_type; + } } } diff --git a/crates/vm/src/stdlib/ctypes/function.rs b/crates/vm/src/stdlib/ctypes/function.rs index e690ffbaf84..d24102ed635 100644 --- a/crates/vm/src/stdlib/ctypes/function.rs +++ b/crates/vm/src/stdlib/ctypes/function.rs @@ -31,6 +31,7 @@ use rustpython_common::lock::PyRwLock; pub(super) const INTERNAL_CAST_ADDR: usize = 1; pub(super) const INTERNAL_STRING_AT_ADDR: usize = 2; pub(super) const INTERNAL_WSTRING_AT_ADDR: usize = 3; +pub(super) const INTERNAL_MEMORYVIEW_AT_ADDR: usize = 4; // Thread-local errno storage for ctypes std::thread_local! { @@ -512,7 +513,17 @@ impl Initializer for PyCFuncPtrType { } #[pyclass(flags(IMMUTABLETYPE), with(Initializer))] -impl PyCFuncPtrType {} +impl PyCFuncPtrType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } +} /// PyCFuncPtr - Function pointer instance /// Saved in _base.buffer @@ -561,6 +572,16 @@ impl Debug for PyCFuncPtr { /// Extract pointer value from a ctypes argument (c_void_p conversion) fn extract_ptr_from_arg(arg: &PyObject, vm: &VirtualMachine) -> PyResult { + // Try CArgObject first - extract the wrapped pointer value, applying offset + if let Some(carg) = arg.downcast_ref::() { + if carg.offset != 0 + && let Some(cdata) = carg.obj.downcast_ref::() + { + let base = cdata.buffer.read().as_ptr() as isize; + return Ok((base + carg.offset) as usize); + } + return extract_ptr_from_arg(&carg.obj, vm); + } // Try to get pointer value from various ctypes types if let Some(ptr) = arg.downcast_ref::() { return Ok(ptr.get_ptr_value()); @@ -655,6 +676,68 @@ fn wstring_at_impl(ptr: usize, size: isize, vm: &VirtualMachine) -> PyResult { } } +/// A buffer wrapping raw memory at a given pointer, for zero-copy memoryview. +#[pyclass(name = "_RawMemoryBuffer", module = "_ctypes")] +#[derive(Debug, PyPayload)] +pub(super) struct RawMemoryBuffer { + ptr: *const u8, + size: usize, + readonly: bool, +} + +// SAFETY: The caller ensures the pointer remains valid +unsafe impl Send for RawMemoryBuffer {} +unsafe impl Sync for RawMemoryBuffer {} + +static RAW_MEMORY_BUFFER_METHODS: crate::protocol::BufferMethods = crate::protocol::BufferMethods { + obj_bytes: |buffer| { + let raw = buffer.obj_as::(); + let slice = unsafe { core::slice::from_raw_parts(raw.ptr, raw.size) }; + rustpython_common::borrow::BorrowedValue::Ref(slice) + }, + obj_bytes_mut: |buffer| { + let raw = buffer.obj_as::(); + let slice = unsafe { core::slice::from_raw_parts_mut(raw.ptr as *mut u8, raw.size) }; + rustpython_common::borrow::BorrowedValueMut::RefMut(slice) + }, + release: |_| {}, + retain: |_| {}, +}; + +#[pyclass(with(AsBuffer))] +impl RawMemoryBuffer {} + +impl AsBuffer for RawMemoryBuffer { + fn as_buffer(zelf: &Py, _vm: &VirtualMachine) -> PyResult { + Ok(PyBuffer::new( + zelf.to_owned().into(), + BufferDescriptor::simple(zelf.size, zelf.readonly), + &RAW_MEMORY_BUFFER_METHODS, + )) + } +} + +/// memoryview_at implementation - create a memoryview from memory at ptr +fn memoryview_at_impl(ptr: usize, size: isize, readonly: bool, vm: &VirtualMachine) -> PyResult { + use crate::builtins::PyMemoryView; + + if ptr == 0 { + return Err(vm.new_value_error("NULL pointer access")); + } + if size < 0 { + return Err(vm.new_value_error("negative size")); + } + let len = size as usize; + let raw_buf = RawMemoryBuffer { + ptr: ptr as *const u8, + size: len, + readonly, + } + .into_pyobject(vm); + let mv = PyMemoryView::from_object(&raw_buf, vm)?; + Ok(mv.into_pyobject(vm)) +} + // cast_check_pointertype fn cast_check_pointertype(ctype: &PyObject, vm: &VirtualMachine) -> bool { use super::pointer::PyCPointerType; @@ -1064,6 +1147,25 @@ fn handle_internal_func(addr: usize, args: &FuncArgs, vm: &VirtualMachine) -> Op })); } + if addr == INTERNAL_MEMORYVIEW_AT_ADDR { + let result: PyResult<(PyObjectRef, PyObjectRef, Option)> = + args.clone().bind(vm); + return Some(result.and_then(|(ptr_arg, size_arg, readonly_arg)| { + let ptr = extract_ptr_from_arg(&ptr_arg, vm)?; + let size_int = size_arg.try_int(vm)?; + let size = size_int + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_value_error("size too large"))?; + let readonly = readonly_arg + .and_then(|r| r.try_int(vm).ok()) + .and_then(|i| i.as_bigint().to_i32()) + .unwrap_or(0) + != 0; + memoryview_at_impl(ptr, size, readonly, vm) + })); + } + None } @@ -2091,7 +2193,7 @@ unsafe extern "C" fn thunk_callback( .map(|s| s.to_string()) .unwrap_or_else(|_| "".to_string()); let msg = format!( - "Exception ignored on calling ctypes callback function {}", + "Exception ignored while calling ctypes callback function {}", repr ); vm.run_unraisable(exc.clone(), Some(msg), vm.ctx.none()); diff --git a/crates/vm/src/stdlib/ctypes/pointer.rs b/crates/vm/src/stdlib/ctypes/pointer.rs index 087a7bfd32d..6000bb57a37 100644 --- a/crates/vm/src/stdlib/ctypes/pointer.rs +++ b/crates/vm/src/stdlib/ctypes/pointer.rs @@ -27,16 +27,24 @@ impl Initializer for PyCPointerType { .downcast() .map_err(|_| vm.new_type_error("expected type"))?; - new_type.check_not_initialized(vm)?; + if new_type.is_initialized() { + return Ok(()); + } // Get the _type_ attribute (element type) - // PyCPointerType_init gets the element type from _type_ attribute let proto = new_type .as_object() .get_attr("_type_", vm) .ok() .and_then(|obj| obj.downcast::().ok()); + // Validate that _type_ has storage info (is a ctypes type) + if let Some(ref proto_type) = proto + && proto_type.stg_info_opt().is_none() + { + return Err(vm.new_type_error(format!("{} must have storage info", proto_type.name()))); + } + // Initialize StgInfo for pointer type let pointer_size = core::mem::size_of::(); let mut stg_info = StgInfo::new(pointer_size, pointer_size); @@ -62,12 +70,31 @@ impl Initializer for PyCPointerType { let _ = new_type.init_type_data(stg_info); + // Cache: set target_type.__pointer_type__ = self (via StgInfo, not as inheritable attr) + if let Ok(type_attr) = new_type.as_object().get_attr("_type_", vm) + && let Ok(target_type) = type_attr.downcast::() + && let Some(mut target_info) = target_type.get_type_data_mut::() + { + let zelf_obj: PyObjectRef = zelf.into(); + target_info.pointer_type = Some(zelf_obj); + } + Ok(()) } } -#[pyclass(flags(IMMUTABLETYPE), with(AsNumber, Initializer))] +#[pyclass(flags(BASETYPE, IMMUTABLETYPE), with(AsNumber, Initializer))] impl PyCPointerType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + #[pymethod] fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { // zelf is the pointer type class that from_param was called on @@ -182,7 +209,12 @@ impl PyCPointerType { } // 4. Set _type_ attribute on the pointer type - zelf.as_object().set_attr("_type_", typ_type, vm)?; + zelf.as_object().set_attr("_type_", typ_type.clone(), vm)?; + + // 5. Cache: set target_type.__pointer_type__ = self (via StgInfo) + if let Some(mut target_info) = typ_type.get_type_data_mut::() { + target_info.pointer_type = Some(zelf.into()); + } Ok(()) } @@ -598,11 +630,12 @@ impl PyCPointer { if type_code.as_deref() == Some("z") && let Some(bytes) = value.downcast_ref::() { - let (converted, ptr_val) = super::base::ensure_z_null_terminated(bytes, vm); + let (kept_alive, ptr_val) = super::base::ensure_z_null_terminated(bytes, vm); unsafe { *(addr as *mut usize) = ptr_val; } - return zelf.0.keep_ref(index as usize, converted, vm); + zelf.0.keep_alive(index as usize, kept_alive); + return zelf.0.keep_ref(index as usize, value.clone(), vm); } else if type_code.as_deref() == Some("Z") && let Some(s) = value.downcast_ref::() { diff --git a/crates/vm/src/stdlib/ctypes/simple.rs b/crates/vm/src/stdlib/ctypes/simple.rs index 48ff52e9cfa..67c07dcb73b 100644 --- a/crates/vm/src/stdlib/ctypes/simple.rs +++ b/crates/vm/src/stdlib/ctypes/simple.rs @@ -18,8 +18,12 @@ use core::fmt::Debug; use num_traits::ToPrimitive; /// Valid type codes for ctypes simple types +#[cfg(windows)] // spell-checker: disable-next-line pub(super) const SIMPLE_TYPE_CHARS: &str = "cbBhHiIlLdfuzZqQPXOv?g"; +#[cfg(not(windows))] +// spell-checker: disable-next-line +pub(super) const SIMPLE_TYPE_CHARS: &str = "cbBhHiIlLdfuzZqQPOv?g"; /// Convert ctypes type code to PEP 3118 format code. /// Some ctypes codes need to be mapped to standard-size codes based on platform. @@ -235,6 +239,16 @@ pub struct PyCSimpleType(PyType); #[pyclass(flags(BASETYPE), with(AsNumber, Initializer))] impl PyCSimpleType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + #[allow(clippy::new_ret_no_self)] #[pymethod] fn new(cls: PyTypeRef, _: OptionalArg, vm: &VirtualMachine) -> PyResult { @@ -327,10 +341,10 @@ impl PyCSimpleType { Some("z") => { // 1. bytes → create CArgObject with null-terminated buffer if let Some(bytes) = value.downcast_ref::() { - let (holder, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); return Ok(CArgObject { tag: b'z', - value: FfiArgValue::OwnedPointer(ptr, holder), + value: FfiArgValue::OwnedPointer(ptr, kept_alive), obj: value.clone(), size: 0, offset: 0, @@ -381,10 +395,10 @@ impl PyCSimpleType { } // 2. bytes → create CArgObject with null-terminated buffer if let Some(bytes) = value.downcast_ref::() { - let (holder, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); return Ok(CArgObject { tag: b'z', - value: FfiArgValue::OwnedPointer(ptr, holder), + value: FfiArgValue::OwnedPointer(ptr, kept_alive), obj: value.clone(), size: 0, offset: 0, @@ -1027,9 +1041,10 @@ impl Constructor for PyCSimple { if let Some(ref v) = init_arg { if _type_ == "z" { if let Some(bytes) = v.downcast_ref::() { - let (converted, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); let buffer = ptr.to_ne_bytes().to_vec(); - let cdata = PyCData::from_bytes(buffer, Some(converted)); + let cdata = PyCData::from_bytes(buffer, Some(v.clone())); + *cdata.base.write() = Some(kept_alive); return PyCSimple(cdata).into_ref_with_type(vm, cls).map(Into::into); } } else if _type_ == "Z" @@ -1258,9 +1273,10 @@ impl PyCSimple { // Handle z/Z types with PyBytes/PyStr separately to avoid memory leak if type_code == "z" { if let Some(bytes) = value.downcast_ref::() { - let (converted, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); *zelf.0.buffer.write() = alloc::borrow::Cow::Owned(ptr.to_ne_bytes().to_vec()); - *zelf.0.objects.write() = Some(converted); + *zelf.0.objects.write() = Some(value); + *zelf.0.base.write() = Some(kept_alive); return Ok(()); } } else if type_code == "Z" diff --git a/crates/vm/src/stdlib/ctypes/structure.rs b/crates/vm/src/stdlib/ctypes/structure.rs index cd36ce85560..c0116d9d76c 100644 --- a/crates/vm/src/stdlib/ctypes/structure.rs +++ b/crates/vm/src/stdlib/ctypes/structure.rs @@ -1,9 +1,9 @@ use super::base::{CDATA_BUFFER_METHODS, PyCData, PyCField, StgInfo, StgInfoFlags}; use crate::builtins::{PyList, PyStr, PyTuple, PyType, PyTypeRef}; use crate::convert::ToPyObject; -use crate::function::FuncArgs; -use crate::function::PySetterValue; +use crate::function::{FuncArgs, OptionalArg, PySetterValue}; use crate::protocol::{BufferDescriptor, PyBuffer, PyNumberMethods}; +use crate::stdlib::warnings; use crate::types::{AsBuffer, AsNumber, Constructor, Initializer, SetAttr}; use crate::{AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; use alloc::borrow::Cow; @@ -103,6 +103,7 @@ impl Initializer for PyCStructType { let mut stg_info = baseinfo.clone(); stg_info.flags &= !StgInfoFlags::DICTFLAG_FINAL; // Clear FINAL in subclass stg_info.initialized = true; + stg_info.pointer_type = None; // Non-inheritable stg_info }); @@ -130,6 +131,16 @@ impl Initializer for PyCStructType { #[pyclass(flags(BASETYPE), with(AsNumber, Constructor, Initializer, SetAttr))] impl PyCStructType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + #[pymethod] fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { // zelf is the structure type class that from_param was called on @@ -154,6 +165,55 @@ impl PyCStructType { ))) } + // CDataType methods - delegated to PyCData implementations + + #[pymethod] + fn from_address(zelf: PyObjectRef, address: isize, vm: &VirtualMachine) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_address(cls, address, vm) + } + + #[pymethod] + fn from_buffer( + zelf: PyObjectRef, + source: PyObjectRef, + offset: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_buffer(cls, source, offset, vm) + } + + #[pymethod] + fn from_buffer_copy( + zelf: PyObjectRef, + source: crate::function::ArgBytesLike, + offset: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_buffer_copy(cls, source, offset, vm) + } + + #[pymethod] + fn in_dll( + zelf: PyObjectRef, + dll: PyObjectRef, + name: crate::builtins::PyStrRef, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::in_dll(cls, dll, name, vm) + } + /// Called when a new Structure subclass is created #[pyclassmethod] fn __init_subclass__(cls: PyTypeRef, vm: &VirtualMachine) -> PyResult<()> { @@ -185,6 +245,24 @@ impl PyCStructType { }; let pack = super::base::get_usize_attr(cls.as_object(), "_pack_", 0, vm)?; + + // Emit DeprecationWarning on non-Windows when _pack_ is set without _layout_ + if pack > 0 && !cfg!(windows) { + let has_layout = cls.as_object().get_attr("_layout_", vm).is_ok(); + if !has_layout { + let base_type_name = "Structure"; + let msg = format!( + "Due to '_pack_', the '{}' {} will use memory layout compatible with \ + MSVC (Windows). If this is intended, set _layout_ to 'ms'. \ + The implicit default is deprecated and slated to become an error in \ + Python 3.19.", + cls.name(), + base_type_name, + ); + warnings::warn(vm.ctx.exceptions.deprecation_warning, msg, 1, vm)?; + } + } + let forced_alignment = super::base::get_usize_attr(cls.as_object(), "_align_", 1, vm)?.max(1); @@ -221,6 +299,11 @@ impl PyCStructType { let mut format = String::from("T{"); let mut last_end = 0usize; // Track end of last field for padding calculation + // Bitfield layout tracking + let mut bitfield_bit_offset: u16 = 0; // Current bit position within bitfield group + let mut last_field_bit_size: u16 = 0; // For MSVC: bit size of previous storage unit + let use_msvc_bitfields = pack > 0; // MSVC layout when _pack_ is set + for (index, field) in fields.iter().enumerate() { let field_tuple = field .downcast_ref::() @@ -336,14 +419,93 @@ impl PyCStructType { .clone() .downcast::() .map_err(|_| vm.new_type_error("_fields_ type must be a ctypes type"))?; - let c_field = PyCField::new(field_type_ref, offset as isize, size as isize, index); + + // Check for bitfield size (optional 3rd element in tuple) + let (c_field, field_advances_offset) = if field_tuple.len() > 2 { + let bit_size_obj = field_tuple.get(2).expect("len checked"); + let bit_size = bit_size_obj + .try_int(vm)? + .as_bigint() + .to_u16() + .ok_or_else(|| { + vm.new_value_error("number of bits invalid for bit field".to_string()) + })?; + has_bitfield = true; + + let type_bits = (size * 8) as u16; + let (advances, bit_offset); + + if use_msvc_bitfields { + // MSVC layout: different types start new storage unit + if bitfield_bit_offset + bit_size > type_bits + || type_bits != last_field_bit_size + { + // Close previous bitfield, start new allocation unit + bitfield_bit_offset = 0; + advances = true; + } else { + advances = false; + } + bit_offset = bitfield_bit_offset; + bitfield_bit_offset += bit_size; + last_field_bit_size = type_bits; + } else { + // GCC System V layout: pack within same type + let fits_in_current = bitfield_bit_offset + bit_size <= type_bits; + advances = if fits_in_current && bitfield_bit_offset > 0 { + false + } else if !fits_in_current { + bitfield_bit_offset = 0; + true + } else { + true + }; + bit_offset = bitfield_bit_offset; + bitfield_bit_offset += bit_size; + } + + // For packed bitfields that share offset, use the same offset as previous + let field_offset = if !advances { + offset - size // Reuse the previous field's offset + } else { + offset + }; + + ( + PyCField::new_bitfield( + name.clone(), + field_type_ref, + field_offset as isize, + size as isize, + bit_size, + bit_offset, + index, + ), + advances, + ) + } else { + bitfield_bit_offset = 0; // Reset on non-bitfield + last_field_bit_size = 0; + ( + PyCField::new( + name.clone(), + field_type_ref, + offset as isize, + size as isize, + index, + ), + true, + ) + }; // Set the CField as a class attribute cls.set_attr(vm.ctx.intern_str(name.clone()), c_field.to_pyobject(vm)); - // Update tracking - last_end = offset + size; - offset += size; + // Update tracking - don't advance offset for packed bitfields + if field_advances_offset { + last_end = offset + size; + offset += size; + } } // Calculate total_align = max(max_align, forced_alignment) @@ -366,6 +528,16 @@ impl PyCStructType { } format.push('}'); + // Check for circular self-reference: if a field of the same type as this + // structure was encountered, it would have marked this type's stginfo as FINAL. + if let Some(stg_info) = cls.get_type_data::() + && stg_info.is_final() + { + return Err( + vm.new_attribute_error("Structure or union cannot contain itself".to_string()) + ); + } + // Store StgInfo with aligned size and total alignment let mut stg_info = StgInfo::new(aligned_size, total_align); stg_info.length = fields.len(); diff --git a/crates/vm/src/stdlib/ctypes/union.rs b/crates/vm/src/stdlib/ctypes/union.rs index aecef18ec7c..fba9a75e955 100644 --- a/crates/vm/src/stdlib/ctypes/union.rs +++ b/crates/vm/src/stdlib/ctypes/union.rs @@ -2,12 +2,13 @@ use super::base::{CDATA_BUFFER_METHODS, StgInfoFlags}; use super::{PyCData, PyCField, StgInfo}; use crate::builtins::{PyList, PyStr, PyTuple, PyType, PyTypeRef}; use crate::convert::ToPyObject; -use crate::function::FuncArgs; -use crate::function::PySetterValue; +use crate::function::{ArgBytesLike, FuncArgs, OptionalArg, PySetterValue}; use crate::protocol::{BufferDescriptor, PyBuffer}; +use crate::stdlib::warnings; use crate::types::{AsBuffer, Constructor, Initializer, SetAttr}; use crate::{AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; use alloc::borrow::Cow; +use num_traits::ToPrimitive; /// Calculate Union type size from _fields_ (max field size) pub(super) fn calculate_union_size(cls: &Py, vm: &VirtualMachine) -> PyResult { @@ -106,6 +107,7 @@ impl Initializer for PyCUnionType { let mut stg_info = baseinfo.clone(); stg_info.flags &= !StgInfoFlags::DICTFLAG_FINAL; // Clear FINAL flag in subclass stg_info.initialized = true; + stg_info.pointer_type = None; // Non-inheritable stg_info }); @@ -163,6 +165,22 @@ impl PyCUnionType { }; let pack = super::base::get_usize_attr(cls.as_object(), "_pack_", 0, vm)?; + + // Emit DeprecationWarning on non-Windows when _pack_ is set without _layout_ + if pack > 0 && !cfg!(windows) { + let has_layout = cls.as_object().get_attr("_layout_", vm).is_ok(); + if !has_layout { + let msg = format!( + "Due to '_pack_', the '{}' Union will use memory layout compatible with \ + MSVC (Windows). If this is intended, set _layout_ to 'ms'. \ + The implicit default is deprecated and slated to become an error in \ + Python 3.19.", + cls.name(), + ); + warnings::warn(vm.ctx.exceptions.deprecation_warning, msg, 1, vm)?; + } + } + let forced_alignment = super::base::get_usize_attr(cls.as_object(), "_align_", 1, vm)?.max(1); @@ -258,7 +276,42 @@ impl PyCUnionType { .clone() .downcast::() .map_err(|_| vm.new_type_error("_fields_ type must be a ctypes type"))?; - let c_field = PyCField::new(field_type_ref, 0, size as isize, index); + + // Check for bitfield size (optional 3rd element in tuple) + // For unions, each field starts fresh (CPython: _layout.py) + let c_field = if field_tuple.len() > 2 { + let bit_size_obj = field_tuple.get(2).expect("len checked"); + let bit_size = bit_size_obj + .try_int(vm)? + .as_bigint() + .to_u16() + .ok_or_else(|| { + vm.new_value_error("number of bits invalid for bit field".to_string()) + })?; + has_bitfield = true; + + // Union fields all start at offset 0, so bit_offset = 0 + let mut bit_offset: u16 = 0; + let type_bits = (size * 8) as u16; + + // Big-endian: bit_offset = type_bits - bit_size + let big_endian = is_swapped != cfg!(target_endian = "big"); + if big_endian && type_bits >= bit_size { + bit_offset = type_bits - bit_size; + } + + PyCField::new_bitfield( + name.clone(), + field_type_ref, + 0, // Union fields always at offset 0 + size as isize, + bit_size, + bit_offset, + index, + ) + } else { + PyCField::new(name.clone(), field_type_ref, 0, size as isize, index) + }; cls.set_attr(vm.ctx.intern_str(name), c_field.to_pyobject(vm)); } @@ -271,6 +324,15 @@ impl PyCUnionType { max_size }; + // Check for circular self-reference + if let Some(stg_info) = cls.get_type_data::() + && stg_info.is_final() + { + return Err( + vm.new_attribute_error("Structure or union cannot contain itself".to_string()) + ); + } + // Store StgInfo with aligned size let mut stg_info = StgInfo::new(aligned_size, total_align); stg_info.length = fields.len(); @@ -299,6 +361,16 @@ impl PyCUnionType { #[pyclass(flags(BASETYPE), with(Constructor, Initializer, SetAttr))] impl PyCUnionType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + #[pymethod] fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { // zelf is the union type class that from_param was called on @@ -344,6 +416,55 @@ impl PyCUnionType { ))) } + // CDataType methods - delegated to PyCData implementations + + #[pymethod] + fn from_address(zelf: PyObjectRef, address: isize, vm: &VirtualMachine) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_address(cls, address, vm) + } + + #[pymethod] + fn from_buffer( + zelf: PyObjectRef, + source: PyObjectRef, + offset: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_buffer(cls, source, offset, vm) + } + + #[pymethod] + fn from_buffer_copy( + zelf: PyObjectRef, + source: ArgBytesLike, + offset: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_buffer_copy(cls, source, offset, vm) + } + + #[pymethod] + fn in_dll( + zelf: PyObjectRef, + dll: PyObjectRef, + name: crate::builtins::PyStrRef, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::in_dll(cls, dll, name, vm) + } + /// Called when a new Union subclass is created #[pyclassmethod] fn __init_subclass__(cls: PyTypeRef, vm: &VirtualMachine) -> PyResult<()> { @@ -383,6 +504,14 @@ impl SetAttr for PyCUnionType { } } + // 2. If _fields_, call process_fields (which checks FINAL internally) + // Check BEFORE writing to dict to avoid storing _fields_ when FINAL + if attr_name.as_str() == "_fields_" + && let PySetterValue::Assign(ref fields_value) = value + { + PyCUnionType::process_fields(pytype, fields_value.clone(), vm)?; + } + // Store in type's attributes dict match &value { PySetterValue::Assign(v) => { @@ -403,13 +532,6 @@ impl SetAttr for PyCUnionType { } } - // 2. If _fields_, call process_fields (which checks FINAL internally) - if attr_name.as_str() == "_fields_" - && let PySetterValue::Assign(fields_value) = value - { - PyCUnionType::process_fields(pytype, fields_value, vm)?; - } - Ok(()) } } diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 22b720a1cd2..3ee6caf2eae 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -827,8 +827,7 @@ mod sys { Ok(exc) => { // PyErr_Display: try traceback._print_exception_bltin first if let Ok(tb_mod) = vm.import("traceback", 0) - && let Ok(print_exc_builtin) = - tb_mod.get_attr("_print_exception_bltin", vm) + && let Ok(print_exc_builtin) = tb_mod.get_attr("_print_exception_bltin", vm) && print_exc_builtin .call((exc.as_object().to_owned(),), vm) .is_ok() diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index a67c0636614..9e33f430945 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -211,7 +211,7 @@ impl VirtualMachine { op: &str, ) -> PyBaseExceptionRef { self.new_type_error(format!( - "'{}' not supported between instances of '{}' and '{}'", + "unsupported operand type(s) for {}: '{}' and '{}'", op, a.class().name(), b.class().name()