diff --git a/.cspell.json b/.cspell.json index 0a93ac35cd5..aae9ed6c853 100644 --- a/.cspell.json +++ b/.cspell.json @@ -147,6 +147,7 @@ // "stat" "FIRMLINK", // CPython internal names + "PYTHONUTF", "sysdict", "settraceallthreads", "setprofileallthreads" diff --git a/crates/vm/src/stdlib/winapi.rs b/crates/vm/src/stdlib/_winapi.rs similarity index 74% rename from crates/vm/src/stdlib/winapi.rs rename to crates/vm/src/stdlib/_winapi.rs index c176568cfc5..98299c671e9 100644 --- a/crates/vm/src/stdlib/winapi.rs +++ b/crates/vm/src/stdlib/_winapi.rs @@ -15,16 +15,17 @@ mod _winapi { windows::{WinHandle, WindowsSysResult}, }; use core::ptr::{null, null_mut}; - use windows_sys::Win32::Foundation::{HANDLE, INVALID_HANDLE_VALUE, MAX_PATH}; + use windows_sys::Win32::Foundation::{HANDLE, MAX_PATH}; #[pyattr] use windows_sys::Win32::{ Foundation::{ - DUPLICATE_CLOSE_SOURCE, DUPLICATE_SAME_ACCESS, ERROR_ALREADY_EXISTS, ERROR_BROKEN_PIPE, - ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_NETNAME_DELETED, ERROR_NO_DATA, - ERROR_NO_SYSTEM_RESOURCES, ERROR_OPERATION_ABORTED, ERROR_PIPE_BUSY, - ERROR_PIPE_CONNECTED, ERROR_SEM_TIMEOUT, GENERIC_READ, GENERIC_WRITE, STILL_ACTIVE, - WAIT_ABANDONED, WAIT_ABANDONED_0, WAIT_OBJECT_0, WAIT_TIMEOUT, + DUPLICATE_CLOSE_SOURCE, DUPLICATE_SAME_ACCESS, ERROR_ACCESS_DENIED, + ERROR_ALREADY_EXISTS, ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, + ERROR_NETNAME_DELETED, ERROR_NO_DATA, ERROR_NO_SYSTEM_RESOURCES, + ERROR_OPERATION_ABORTED, ERROR_PIPE_BUSY, ERROR_PIPE_CONNECTED, + ERROR_PRIVILEGE_NOT_HELD, ERROR_SEM_TIMEOUT, GENERIC_READ, GENERIC_WRITE, STILL_ACTIVE, + WAIT_ABANDONED_0, WAIT_OBJECT_0, WAIT_TIMEOUT, }, Globalization::{ LCMAP_FULLWIDTH, LCMAP_HALFWIDTH, LCMAP_HIRAGANA, LCMAP_KATAKANA, @@ -32,79 +33,45 @@ mod _winapi { LCMAP_TRADITIONAL_CHINESE, LCMAP_UPPERCASE, }, Storage::FileSystem::{ - COPY_FILE_ALLOW_DECRYPTED_DESTINATION, - COPY_FILE_COPY_SYMLINK, - COPY_FILE_FAIL_IF_EXISTS, - COPY_FILE_NO_BUFFERING, - COPY_FILE_NO_OFFLOAD, - COPY_FILE_OPEN_SOURCE_FOR_WRITE, - COPY_FILE_REQUEST_COMPRESSED_TRAFFIC, - COPY_FILE_REQUEST_SECURITY_PRIVILEGES, - COPY_FILE_RESTARTABLE, - COPY_FILE_RESUME_FROM_PAUSE, - COPYFILE2_CALLBACK_CHUNK_FINISHED, - COPYFILE2_CALLBACK_CHUNK_STARTED, - COPYFILE2_CALLBACK_ERROR, - COPYFILE2_CALLBACK_POLL_CONTINUE, - COPYFILE2_CALLBACK_STREAM_FINISHED, - COPYFILE2_CALLBACK_STREAM_STARTED, - COPYFILE2_PROGRESS_CANCEL, - COPYFILE2_PROGRESS_CONTINUE, - COPYFILE2_PROGRESS_PAUSE, - COPYFILE2_PROGRESS_QUIET, - COPYFILE2_PROGRESS_STOP, - CREATE_ALWAYS, - // CreateFile constants - CREATE_NEW, - FILE_ATTRIBUTE_NORMAL, - FILE_FLAG_BACKUP_SEMANTICS, - FILE_FLAG_DELETE_ON_CLOSE, - FILE_FLAG_FIRST_PIPE_INSTANCE, - FILE_FLAG_NO_BUFFERING, - FILE_FLAG_OPEN_REPARSE_POINT, - FILE_FLAG_OVERLAPPED, - FILE_FLAG_POSIX_SEMANTICS, - FILE_FLAG_RANDOM_ACCESS, - FILE_FLAG_SEQUENTIAL_SCAN, - FILE_FLAG_WRITE_THROUGH, - FILE_GENERIC_READ, - FILE_GENERIC_WRITE, - FILE_SHARE_DELETE, - FILE_SHARE_READ, - FILE_SHARE_WRITE, - FILE_TYPE_CHAR, - FILE_TYPE_DISK, - FILE_TYPE_PIPE, - FILE_TYPE_REMOTE, - FILE_TYPE_UNKNOWN, - OPEN_ALWAYS, - OPEN_EXISTING, - PIPE_ACCESS_DUPLEX, - PIPE_ACCESS_INBOUND, - SYNCHRONIZE, - TRUNCATE_EXISTING, + COPY_FILE_ALLOW_DECRYPTED_DESTINATION, COPY_FILE_COPY_SYMLINK, + COPY_FILE_FAIL_IF_EXISTS, COPY_FILE_NO_BUFFERING, COPY_FILE_NO_OFFLOAD, + COPY_FILE_OPEN_SOURCE_FOR_WRITE, COPY_FILE_REQUEST_COMPRESSED_TRAFFIC, + COPY_FILE_REQUEST_SECURITY_PRIVILEGES, COPY_FILE_RESTARTABLE, + COPY_FILE_RESUME_FROM_PAUSE, COPYFILE2_CALLBACK_CHUNK_FINISHED, + COPYFILE2_CALLBACK_CHUNK_STARTED, COPYFILE2_CALLBACK_ERROR, + COPYFILE2_CALLBACK_POLL_CONTINUE, COPYFILE2_CALLBACK_STREAM_FINISHED, + COPYFILE2_CALLBACK_STREAM_STARTED, COPYFILE2_PROGRESS_CANCEL, + COPYFILE2_PROGRESS_CONTINUE, COPYFILE2_PROGRESS_PAUSE, COPYFILE2_PROGRESS_QUIET, + COPYFILE2_PROGRESS_STOP, FILE_FLAG_FIRST_PIPE_INSTANCE, FILE_FLAG_OVERLAPPED, + FILE_GENERIC_READ, FILE_GENERIC_WRITE, FILE_TYPE_CHAR, FILE_TYPE_DISK, FILE_TYPE_PIPE, + FILE_TYPE_REMOTE, FILE_TYPE_UNKNOWN, OPEN_EXISTING, PIPE_ACCESS_DUPLEX, + PIPE_ACCESS_INBOUND, SYNCHRONIZE, }, System::{ Console::{STD_ERROR_HANDLE, STD_INPUT_HANDLE, STD_OUTPUT_HANDLE}, Memory::{ - FILE_MAP_ALL_ACCESS, MEM_COMMIT, MEM_FREE, MEM_IMAGE, MEM_MAPPED, MEM_PRIVATE, + FILE_MAP_ALL_ACCESS, FILE_MAP_COPY, FILE_MAP_EXECUTE, FILE_MAP_READ, + FILE_MAP_WRITE, MEM_COMMIT, MEM_FREE, MEM_IMAGE, MEM_MAPPED, MEM_PRIVATE, MEM_RESERVE, PAGE_EXECUTE, PAGE_EXECUTE_READ, PAGE_EXECUTE_READWRITE, PAGE_EXECUTE_WRITECOPY, PAGE_GUARD, PAGE_NOACCESS, PAGE_NOCACHE, PAGE_READONLY, PAGE_READWRITE, PAGE_WRITECOMBINE, PAGE_WRITECOPY, SEC_COMMIT, SEC_IMAGE, SEC_LARGE_PAGES, SEC_NOCACHE, SEC_RESERVE, SEC_WRITECOMBINE, }, Pipes::{ - NMPWAIT_NOWAIT, NMPWAIT_USE_DEFAULT_WAIT, NMPWAIT_WAIT_FOREVER, - PIPE_READMODE_MESSAGE, PIPE_TYPE_MESSAGE, PIPE_UNLIMITED_INSTANCES, PIPE_WAIT, + NMPWAIT_WAIT_FOREVER, PIPE_READMODE_MESSAGE, PIPE_TYPE_MESSAGE, + PIPE_UNLIMITED_INSTANCES, PIPE_WAIT, }, SystemServices::LOCALE_NAME_MAX_LENGTH, Threading::{ ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS, CREATE_BREAKAWAY_FROM_JOB, CREATE_DEFAULT_ERROR_MODE, CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, CREATE_NO_WINDOW, DETACHED_PROCESS, HIGH_PRIORITY_CLASS, - IDLE_PRIORITY_CLASS, INFINITE, NORMAL_PRIORITY_CLASS, PROCESS_DUP_HANDLE, - REALTIME_PRIORITY_CLASS, STARTF_FORCEOFFFEEDBACK, STARTF_FORCEONFEEDBACK, - STARTF_USESHOWWINDOW, STARTF_USESTDHANDLES, + IDLE_PRIORITY_CLASS, INFINITE, NORMAL_PRIORITY_CLASS, PROCESS_ALL_ACCESS, + PROCESS_DUP_HANDLE, REALTIME_PRIORITY_CLASS, STARTF_FORCEOFFFEEDBACK, + STARTF_FORCEONFEEDBACK, STARTF_PREVENTPINNING, STARTF_RUNFULLSCREEN, + STARTF_TITLEISAPPID, STARTF_TITLEISLINKNAME, STARTF_UNTRUSTEDSOURCE, + STARTF_USECOUNTCHARS, STARTF_USEFILLATTRIBUTE, STARTF_USEHOTKEY, + STARTF_USEPOSITION, STARTF_USESHOWWINDOW, STARTF_USESIZE, STARTF_USESTDHANDLES, }, }, UI::WindowsAndMessaging::SW_HIDE, @@ -113,6 +80,12 @@ mod _winapi { #[pyattr] const NULL: isize = 0; + #[pyattr] + const INVALID_HANDLE_VALUE: isize = -1; + + #[pyattr] + const COPY_FILE_DIRECTORY: u32 = 0x00000080; + #[pyfunction] fn CloseHandle(handle: WinHandle) -> WindowsSysResult { WindowsSysResult(unsafe { windows_sys::Win32::Foundation::CloseHandle(handle.0) }) @@ -150,7 +123,7 @@ mod _winapi { ) }; - if handle == INVALID_HANDLE_VALUE { + if handle == windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE { return Err(vm.new_last_os_error()); } @@ -163,7 +136,7 @@ mod _winapi { vm: &VirtualMachine, ) -> PyResult> { let handle = unsafe { windows_sys::Win32::System::Console::GetStdHandle(std_handle) }; - if handle == INVALID_HANDLE_VALUE { + if handle == windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE { return Err(vm.new_last_os_error()); } Ok(if handle.is_null() { @@ -306,9 +279,9 @@ mod _winapi { } si_attr!(dwFlags); si_attr!(wShowWindow); - si_attr!(hStdInput, usize); - si_attr!(hStdOutput, usize); - si_attr!(hStdError, usize); + si_attr!(hStdInput, isize); + si_attr!(hStdOutput, isize); + si_attr!(hStdError, isize); let mut env = args .env_mapping @@ -638,18 +611,6 @@ mod _winapi { }) } - // TODO: ctypes.LibraryLoader.LoadLibrary - #[allow(dead_code)] - fn LoadLibrary(path: PyStrRef, vm: &VirtualMachine) -> PyResult { - let path_wide = path.as_wtf8().to_wide_with_nul(); - let handle = - unsafe { windows_sys::Win32::System::LibraryLoader::LoadLibraryW(path_wide.as_ptr()) }; - if handle.is_null() { - return Err(vm.new_runtime_error("LoadLibrary failed")); - } - Ok(handle as isize) - } - #[pyfunction] fn GetModuleFileName(handle: isize, vm: &VirtualMachine) -> PyResult { let mut path: Vec = vec![0; MAX_PATH as usize]; @@ -675,7 +636,7 @@ mod _winapi { inherit_handle: bool, name: PyStrRef, vm: &VirtualMachine, - ) -> PyResult { + ) -> PyResult { let name_wide = name.as_wtf8().to_wide_with_nul(); let handle = unsafe { windows_sys::Win32::System::Threading::OpenMutexW( @@ -684,23 +645,29 @@ mod _winapi { name_wide.as_ptr(), ) }; - if handle == INVALID_HANDLE_VALUE { + if handle.is_null() { return Err(vm.new_last_os_error()); } - Ok(handle as _) + Ok(WinHandle(handle)) } #[pyfunction] - fn ReleaseMutex(handle: isize) -> WindowsSysResult { - WindowsSysResult(unsafe { - windows_sys::Win32::System::Threading::ReleaseMutex(handle as _) - }) + fn ReleaseMutex(handle: WinHandle) -> WindowsSysResult { + WindowsSysResult(unsafe { windows_sys::Win32::System::Threading::ReleaseMutex(handle.0) }) } // LOCALE_NAME_INVARIANT is an empty string in Windows API #[pyattr] const LOCALE_NAME_INVARIANT: &str = ""; + #[pyattr] + const LOCALE_NAME_SYSTEM_DEFAULT: &str = "!x-sys-default-locale"; + + #[pyattr(name = "LOCALE_NAME_USER_DEFAULT")] + fn locale_name_user_default(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.none() + } + /// LCMapStringEx - Map a string to another string using locale-specific rules /// This is used by ntpath.normcase() for proper Windows case conversion #[pyfunction] @@ -815,7 +782,7 @@ mod _winapi { ) }; - if handle == INVALID_HANDLE_VALUE { + if handle == windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE { return Err(vm.new_last_os_error()); } @@ -1193,7 +1160,7 @@ mod _winapi { initial_state: bool, name: Option, vm: &VirtualMachine, - ) -> PyResult> { + ) -> PyResult { use windows_sys::Win32::System::Threading::CreateEventW as WinCreateEventW; let _ = security_attributes; // Ignored, always NULL @@ -1210,15 +1177,11 @@ mod _winapi { ) }; - if handle == INVALID_HANDLE_VALUE { - return Err(vm.new_last_os_error()); - } - if handle.is_null() { - return Ok(None); + return Err(vm.new_last_os_error()); } - Ok(Some(WinHandle(handle))) + Ok(WinHandle(handle)) } /// SetEvent - Set the specified event object to the signaled state. @@ -1235,18 +1198,24 @@ mod _winapi { Ok(()) } - /// WriteFile - Write data to a file or I/O device. - #[pyfunction] - fn WriteFile( + #[derive(FromArgs)] + struct WriteFileArgs { + #[pyarg(positional)] handle: WinHandle, + #[pyarg(positional)] buffer: crate::function::ArgBytesLike, - use_overlapped: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { + #[pyarg(named, default = false)] + overlapped: bool, + } + + /// WriteFile - Write data to a file or I/O device. + #[pyfunction] + fn WriteFile(args: WriteFileArgs, vm: &VirtualMachine) -> PyResult { use windows_sys::Win32::Storage::FileSystem::WriteFile as WinWriteFile; - let use_overlapped = use_overlapped.unwrap_or(false); - let buf = buffer.borrow_buf(); + let handle = args.handle; + let use_overlapped = args.overlapped; + let buf = args.buffer.borrow_buf(); let len = core::cmp::min(buf.len(), u32::MAX as usize) as u32; if use_overlapped { @@ -1316,6 +1285,184 @@ mod _winapi { .into()) } + #[derive(FromArgs)] + struct ReadFileArgs { + #[pyarg(positional)] + handle: WinHandle, + #[pyarg(positional)] + size: u32, + #[pyarg(named, default = false)] + overlapped: bool, + } + + /// ReadFile - Read data from a file or I/O device. + #[pyfunction] + fn ReadFile(args: ReadFileArgs, vm: &VirtualMachine) -> PyResult { + use windows_sys::Win32::Storage::FileSystem::ReadFile as WinReadFile; + + let handle = args.handle; + let size = args.size; + let use_overlapped = args.overlapped; + + if use_overlapped { + use windows_sys::Win32::Foundation::ERROR_IO_PENDING; + + let ov = Overlapped::new_with_handle(handle.0); + let err = { + let mut inner = ov.inner.lock(); + inner.read_buffer = Some(vec![0u8; size as usize]); + let read_buf = inner.read_buffer.as_mut().unwrap(); + let mut nread: u32 = 0; + let ret = unsafe { + WinReadFile( + handle.0, + read_buf.as_mut_ptr() as *mut _, + size, + &mut nread, + &mut inner.overlapped, + ) + }; + + let err = if ret == 0 { + unsafe { windows_sys::Win32::Foundation::GetLastError() } + } else { + 0 + }; + + if ret == 0 && err != ERROR_IO_PENDING && err != ERROR_MORE_DATA { + return Err(vm.new_last_os_error()); + } + if ret == 0 && err == ERROR_IO_PENDING { + inner.pending = true; + } + + err + }; + let result = vm + .ctx + .new_tuple(vec![ov.into_pyobject(vm), vm.ctx.new_int(err).into()]); + return Ok(result.into()); + } + + let mut buf = vec![0u8; size as usize]; + let mut nread: u32 = 0; + let ret = unsafe { + WinReadFile( + handle.0, + buf.as_mut_ptr() as *mut _, + size, + &mut nread, + null_mut(), + ) + }; + let err = if ret == 0 { + unsafe { windows_sys::Win32::Foundation::GetLastError() } + } else { + 0 + }; + if ret == 0 && err != ERROR_MORE_DATA { + return Err(vm.new_last_os_error()); + } + buf.truncate(nread as usize); + Ok(vm + .ctx + .new_tuple(vec![ + vm.ctx.new_bytes(buf).into(), + vm.ctx.new_int(err).into(), + ]) + .into()) + } + + /// SetNamedPipeHandleState - Set the read mode and other options of a named pipe. + #[pyfunction] + fn SetNamedPipeHandleState( + named_pipe: WinHandle, + mode: PyObjectRef, + max_collection_count: PyObjectRef, + collect_data_timeout: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + use windows_sys::Win32::System::Pipes::SetNamedPipeHandleState as WinSetNamedPipeHandleState; + + let mut dw_args: [u32; 3] = [0; 3]; + let mut p_args: [*mut u32; 3] = [null_mut(); 3]; + + let objs = [&mode, &max_collection_count, &collect_data_timeout]; + for (i, obj) in objs.iter().enumerate() { + if !vm.is_none(obj) { + dw_args[i] = u32::try_from_object(vm, (*obj).clone())?; + p_args[i] = &mut dw_args[i]; + } + } + + let ret = + unsafe { WinSetNamedPipeHandleState(named_pipe.0, p_args[0], p_args[1], p_args[2]) }; + + if ret == 0 { + return Err(vm.new_last_os_error()); + } + Ok(()) + } + + /// ResetEvent - Reset the specified event object to the nonsignaled state. + #[pyfunction] + fn ResetEvent(event: WinHandle, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::System::Threading::ResetEvent as WinResetEvent; + + let ret = unsafe { WinResetEvent(event.0) }; + if ret == 0 { + return Err(vm.new_last_os_error()); + } + Ok(()) + } + + /// CreateMutexW - Create or open a named or unnamed mutex object. + #[pyfunction] + fn CreateMutexW( + security_attributes: isize, + initial_owner: bool, + name: Option, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::System::Threading::CreateMutexW as WinCreateMutexW; + + let _ = security_attributes; + let name_wide = name.map(|n| n.as_wtf8().to_wide_with_nul()); + let name_ptr = name_wide.as_ref().map_or(null(), |n| n.as_ptr()); + + let handle = unsafe { WinCreateMutexW(null(), i32::from(initial_owner), name_ptr) }; + + if handle.is_null() { + return Err(vm.new_last_os_error()); + } + Ok(WinHandle(handle)) + } + + /// OpenEventW - Open an existing named event object. + #[pyfunction] + fn OpenEventW( + desired_access: u32, + inherit_handle: bool, + name: PyStrRef, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::System::Threading::OpenEventW as WinOpenEventW; + + let name_wide = name.as_wtf8().to_wide_with_nul(); + let handle = unsafe { + WinOpenEventW( + desired_access, + i32::from(inherit_handle), + name_wide.as_ptr(), + ) + }; + + if handle.is_null() { + return Err(vm.new_last_os_error()); + } + Ok(WinHandle(handle)) + } + const MAXIMUM_WAIT_OBJECTS: usize = 64; /// BatchedWaitForMultipleObjects - Wait for multiple handles, supporting more than 64. @@ -1642,4 +1789,272 @@ mod _winapi { Ok(vm.ctx.new_list(triggered_indices).into()) } } + + /// CreateFileMapping - Create or open a named or unnamed file mapping object. + #[pyfunction] + fn CreateFileMapping( + file_handle: WinHandle, + _security_attributes: PyObjectRef, + protect: u32, + max_size_high: u32, + max_size_low: u32, + name: Option, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::System::Memory::CreateFileMappingW; + + let name_wide = name.as_ref().map(|n| n.as_wtf8().to_wide_with_nul()); + let name_ptr = name_wide.as_ref().map_or(null(), |n| n.as_ptr()); + + let handle = unsafe { + CreateFileMappingW( + file_handle.0, + null(), + protect, + max_size_high, + max_size_low, + name_ptr, + ) + }; + + if handle.is_null() { + return Err(vm.new_last_os_error()); + } + Ok(WinHandle(handle)) + } + + /// OpenFileMapping - Open a named file mapping object. + #[pyfunction] + fn OpenFileMapping( + desired_access: u32, + inherit_handle: bool, + name: PyStrRef, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::System::Memory::OpenFileMappingW; + + let name_wide = name.as_wtf8().to_wide_with_nul(); + let handle = unsafe { + OpenFileMappingW( + desired_access, + i32::from(inherit_handle), + name_wide.as_ptr(), + ) + }; + + if handle.is_null() { + return Err(vm.new_last_os_error()); + } + Ok(WinHandle(handle)) + } + + /// MapViewOfFile - Map a view of a file mapping into the address space. + #[pyfunction] + fn MapViewOfFile( + file_map: WinHandle, + desired_access: u32, + file_offset_high: u32, + file_offset_low: u32, + number_bytes: usize, + vm: &VirtualMachine, + ) -> PyResult { + let address = unsafe { + windows_sys::Win32::System::Memory::MapViewOfFile( + file_map.0, + desired_access, + file_offset_high, + file_offset_low, + number_bytes, + ) + }; + + let ptr = address.Value; + if ptr.is_null() { + return Err(vm.new_last_os_error()); + } + Ok(ptr as isize) + } + + /// UnmapViewOfFile - Unmap a mapped view of a file. + #[pyfunction] + fn UnmapViewOfFile(address: isize, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::System::Memory::MEMORY_MAPPED_VIEW_ADDRESS; + + let view = MEMORY_MAPPED_VIEW_ADDRESS { + Value: address as *mut core::ffi::c_void, + }; + let ret = unsafe { windows_sys::Win32::System::Memory::UnmapViewOfFile(view) }; + + if ret == 0 { + return Err(vm.new_last_os_error()); + } + Ok(()) + } + + /// VirtualQuerySize - Return the size of a memory region. + #[pyfunction] + fn VirtualQuerySize(address: isize, vm: &VirtualMachine) -> PyResult { + use windows_sys::Win32::System::Memory::{MEMORY_BASIC_INFORMATION, VirtualQuery}; + + let mut mbi: MEMORY_BASIC_INFORMATION = unsafe { core::mem::zeroed() }; + let ret = unsafe { + VirtualQuery( + address as *const core::ffi::c_void, + &mut mbi, + core::mem::size_of::(), + ) + }; + + if ret == 0 { + return Err(vm.new_last_os_error()); + } + Ok(mbi.RegionSize) + } + + /// CopyFile2 - Copy a file with extended parameters. + #[pyfunction] + fn CopyFile2( + existing_file_name: PyStrRef, + new_file_name: PyStrRef, + flags: u32, + _progress_routine: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult<()> { + use windows_sys::Win32::Storage::FileSystem::{ + COPYFILE2_EXTENDED_PARAMETERS, CopyFile2 as WinCopyFile2, + }; + + let src_wide = existing_file_name.as_wtf8().to_wide_with_nul(); + let dst_wide = new_file_name.as_wtf8().to_wide_with_nul(); + + let mut params: COPYFILE2_EXTENDED_PARAMETERS = unsafe { core::mem::zeroed() }; + params.dwSize = core::mem::size_of::() as u32; + params.dwCopyFlags = flags; + + let hr = unsafe { WinCopyFile2(src_wide.as_ptr(), dst_wide.as_ptr(), ¶ms) }; + + if hr < 0 { + // HRESULT failure - convert to Windows error code + let err = if (hr as u32 >> 16) == 0x8007 { + (hr as u32) & 0xFFFF + } else { + hr as u32 + }; + return Err(std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm)); + } + Ok(()) + } + + /// _mimetypes_read_windows_registry - Read MIME type associations from registry. + #[pyfunction] + fn _mimetypes_read_windows_registry( + on_type_read: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + use windows_sys::Win32::System::Registry::{ + HKEY, HKEY_CLASSES_ROOT, KEY_READ, REG_SZ, RegCloseKey, RegEnumKeyExW, RegOpenKeyExW, + RegQueryValueExW, + }; + + let mut hkcr: HKEY = null_mut() as HKEY; + let err = unsafe { RegOpenKeyExW(HKEY_CLASSES_ROOT, null(), 0, KEY_READ, &mut hkcr) }; + if err != 0 { + return Err(vm.new_os_error(err as i32)); + } + scopeguard::defer! { unsafe { RegCloseKey(hkcr) }; } + + let mut i: u32 = 0; + let mut entries: Vec<(String, String)> = Vec::new(); + + loop { + let mut ext_buf = [0u16; 128]; + let mut cch_ext: u32 = ext_buf.len() as u32; + + let err = unsafe { + RegEnumKeyExW( + hkcr, + i, + ext_buf.as_mut_ptr(), + &mut cch_ext, + null_mut(), + null_mut(), + null_mut(), + null_mut(), + ) + }; + i += 1; + + if err == windows_sys::Win32::Foundation::ERROR_NO_MORE_ITEMS { + break; + } + if err != 0 && err != windows_sys::Win32::Foundation::ERROR_MORE_DATA { + return Err(vm.new_os_error(err as i32)); + } + + // Only process keys starting with '.' + if cch_ext == 0 || ext_buf[0] != b'.' as u16 { + continue; + } + + let ext_wide = &ext_buf[..cch_ext as usize]; + + // Open subkey to read Content Type + let mut subkey: HKEY = null_mut() as HKEY; + let err = unsafe { RegOpenKeyExW(hkcr, ext_buf.as_ptr(), 0, KEY_READ, &mut subkey) }; + if err == windows_sys::Win32::Foundation::ERROR_FILE_NOT_FOUND + || err == windows_sys::Win32::Foundation::ERROR_ACCESS_DENIED + { + continue; + } + if err != 0 { + return Err(vm.new_os_error(err as i32)); + } + + let content_type_key: Vec = "Content Type\0".encode_utf16().collect(); + let mut type_buf = [0u16; 256]; + let mut cb_type: u32 = (type_buf.len() * 2) as u32; + let mut reg_type: u32 = 0; + + let err = unsafe { + RegQueryValueExW( + subkey, + content_type_key.as_ptr(), + null_mut(), + &mut reg_type, + type_buf.as_mut_ptr() as *mut u8, + &mut cb_type, + ) + }; + unsafe { RegCloseKey(subkey) }; + + if err != 0 || reg_type != REG_SZ || cb_type == 0 { + continue; + } + + // Convert wide strings to Rust strings + let type_len = (cb_type as usize / 2).saturating_sub(1); // exclude null terminator + let type_str = String::from_utf16_lossy(&type_buf[..type_len]); + let ext_str = String::from_utf16_lossy(ext_wide); + + if type_str.is_empty() { + continue; + } + + entries.push((type_str, ext_str)); + + // Flush buffer periodically to call Python callback + if entries.len() >= 64 { + for (mime_type, ext) in entries.drain(..) { + on_type_read.call((vm.ctx.new_str(mime_type), vm.ctx.new_str(ext)), vm)?; + } + } + } + + // Process remaining entries + for (mime_type, ext) in entries { + on_type_read.call((vm.ctx.new_str(mime_type), vm.ctx.new_str(ext)), vm)?; + } + + Ok(()) + } } diff --git a/crates/vm/src/stdlib/mod.rs b/crates/vm/src/stdlib/mod.rs index e3bf42a4f77..f4f266bf161 100644 --- a/crates/vm/src/stdlib/mod.rs +++ b/crates/vm/src/stdlib/mod.rs @@ -64,6 +64,7 @@ mod _wmi; pub(crate) mod signal; pub mod sys; #[cfg(all(feature = "host_env", windows))] +#[path = "_winapi.rs"] mod winapi; #[cfg(all(feature = "host_env", windows))] mod winreg; diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 492324277e9..915da5acd50 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -1588,7 +1588,11 @@ mod sys { hash_randomization: settings.hash_seed.is_none() as u8, isolated: settings.isolated as u8, dev_mode: settings.dev_mode, - utf8_mode: settings.utf8_mode, + utf8_mode: if settings.utf8_mode < 0 { + 1 + } else { + settings.utf8_mode as u8 + }, int_max_str_digits: settings.int_max_str_digits, safe_path: settings.safe_path, warn_default_encoding: settings.warn_default_encoding as u8, diff --git a/crates/vm/src/vm/setting.rs b/crates/vm/src/vm/setting.rs index 1a5ef9efa8f..0563e7131ac 100644 --- a/crates/vm/src/vm/setting.rs +++ b/crates/vm/src/vm/setting.rs @@ -124,7 +124,7 @@ pub struct Settings { pub stdio_encoding: Option, /// PYTHONIOENCODING - stdio error handler pub stdio_errors: Option, - pub utf8_mode: u8, + pub utf8_mode: i8, /// --check-hash-based-pycs pub check_hash_pycs_mode: CheckHashPycsMode, @@ -211,7 +211,7 @@ impl Default for Settings { allow_external_library: cfg!(feature = "importlib"), stdio_encoding: None, stdio_errors: None, - utf8_mode: 1, + utf8_mode: -1, int_max_str_digits: 4300, #[cfg(feature = "flame-it")] profile_output: None, diff --git a/crates/vm/src/windows.rs b/crates/vm/src/windows.rs index 017384e2f5c..ec9ac8f18fb 100644 --- a/crates/vm/src/windows.rs +++ b/crates/vm/src/windows.rs @@ -61,7 +61,7 @@ impl ToPyResult for WindowsSysResult { } } -type HandleInt = usize; // TODO: change to isize when fully ported to windows-rs +type HandleInt = isize; impl TryFromObject for WinHandle { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { diff --git a/extra_tests/snippets/syntax_non_utf8.py b/extra_tests/snippets/syntax_non_utf8.py index ad93d5b33c1..8f11078bc07 100644 --- a/extra_tests/snippets/syntax_non_utf8.py +++ b/extra_tests/snippets/syntax_non_utf8.py @@ -5,5 +5,5 @@ dir_path = os.path.dirname(os.path.realpath(__file__)) with assert_raises(SyntaxError): - with open(os.path.join(dir_path , "non_utf8.txt")) as f: + with open(os.path.join(dir_path , "non_utf8.txt"), encoding="latin-1") as f: eval(f.read()) diff --git a/src/settings.rs b/src/settings.rs index 79c67dafce0..5233cf98d49 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -260,6 +260,24 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { settings.check_hash_pycs_mode = args.check_hash_based_pycs; + if let Some(val) = get_env("PYTHONUTF8") + && let Some(val_str) = val.to_str() + && !val_str.is_empty() + { + settings.utf8_mode = match val_str { + "1" => 1, + "0" => 0, + _ => { + error!( + "Fatal Python error: config_init_utf8_mode: \ + PYTHONUTF8=N: N is missing or invalid\n\ + Python runtime state: preinitialized" + ); + std::process::exit(1); + } + }; + } + let xopts = args.implementation_option.into_iter().map(|s| { let (name, value) = match s.split_once('=') { Some((name, value)) => (name.to_owned(), Some(value)), @@ -320,6 +338,13 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { }); settings.xoptions.extend(xopts); + // Resolve utf8_mode if not explicitly set by PYTHONUTF8 or -X utf8. + // Default to UTF-8 mode since RustPython's locale encoding detection + // is incomplete. Users can set PYTHONUTF8=0 or -X utf8=0 to disable. + if settings.utf8_mode < 0 { + settings.utf8_mode = 1; + } + settings.warn_default_encoding = settings.warn_default_encoding || env_bool("PYTHONWARNDEFAULTENCODING"); settings.faulthandler = settings.faulthandler || env_bool("PYTHONFAULTHANDLER");