diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 0b634ea5dd0..6135cb75d76 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -136,46 +136,6 @@ def test_c_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_buffers_error() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_builtin_functions() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_bytearray_memoization() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bytes_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_bytes_memoization() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_in_band_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_in_band_buffers() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_oob_buffers() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_oob_buffers_writable_to_readonly() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_buffers_error() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_builtin_functions() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes @@ -201,7 +161,6 @@ def test_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_oob_buffers_writable_to_readonly() - class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests, BigmemPickleTests, unittest.TestCase): @@ -250,11 +209,6 @@ def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_buffers_error() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_builtin_functions() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index f4ce1c75b87..6ec8b348f58 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -67,11 +67,6 @@ def test_optimize_binput_and_memoize(self): def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_buffers_error() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_builtin_functions() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes diff --git a/benches/execution.rs b/benches/execution.rs index bf1bcfcc3db..2816f0e8201 100644 --- a/benches/execution.rs +++ b/benches/execution.rs @@ -24,10 +24,9 @@ fn bench_rustpython_code(b: &mut Bencher, name: &str, source: &str) { settings.path_list.push("Lib/".to_string()); settings.write_bytecode = false; settings.user_site_directory = false; - Interpreter::with_init(settings, |vm| { - vm.add_native_modules(rustpython_stdlib::get_module_inits()); - }) - .enter(|vm| { + let builder = Interpreter::builder(settings); + let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + builder.add_native_modules(&defs).build().enter(|vm| { // Note: bench_cpython is both compiling and executing the code. // As such we compile the code in the benchmark loop as well. b.iter(|| { diff --git a/benches/microbenchmarks.rs b/benches/microbenchmarks.rs index 98993b41543..ba5dcd6c2ec 100644 --- a/benches/microbenchmarks.rs +++ b/benches/microbenchmarks.rs @@ -113,12 +113,10 @@ fn bench_rustpython_code(group: &mut BenchmarkGroup, bench: &MicroBenc settings.write_bytecode = false; settings.user_site_directory = false; - Interpreter::with_init(settings, |vm| { - for (name, init) in rustpython_stdlib::get_module_inits() { - vm.add_native_module(name, init); - } - }) - .enter(|vm| { + let builder = Interpreter::builder(settings); + let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + let interp = builder.add_native_modules(&defs).build(); + interp.enter(|vm| { let setup_code = vm .compile(&bench.setup, Mode::Exec, bench.name.to_owned()) .expect("Error compiling setup code"); diff --git a/crates/common/src/crt_fd.rs b/crates/common/src/crt_fd.rs index 1902a362e32..7c93a30bb65 100644 --- a/crates/common/src/crt_fd.rs +++ b/crates/common/src/crt_fd.rs @@ -5,8 +5,10 @@ use alloc::fmt; use core::cmp; use std::{ffi, io}; +#[cfg(unix)] +use std::os::fd::AsFd; #[cfg(not(windows))] -use std::os::fd::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; +use std::os::fd::{AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; #[cfg(windows)] use std::os::windows::io::BorrowedHandle; diff --git a/crates/derive-impl/src/pymodule.rs b/crates/derive-impl/src/pymodule.rs index 3689ac97fd8..278aab37dbc 100644 --- a/crates/derive-impl/src/pymodule.rs +++ b/crates/derive-impl/src/pymodule.rs @@ -58,7 +58,7 @@ struct ModuleContext { name: String, function_items: FunctionNursery, attribute_items: ItemNursery, - has_extend_module: bool, // TODO: check if `fn extend_module` exists + has_module_exec: bool, errors: Vec, } @@ -82,6 +82,12 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result Result = #doc; }, parse_quote! { - pub(crate) fn __module_def( + pub(crate) fn module_def( ctx: &::rustpython_vm::Context, ) -> &'static ::rustpython_vm::builtins::PyModuleDef { DEF.get_or_init(|| { @@ -133,29 +139,17 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result ::rustpython_vm::PyRef<::rustpython_vm::builtins::PyModule> { - use ::rustpython_vm::PyPayload; - let module = ::rustpython_vm::builtins::PyModule::from_def(__module_def(&vm.ctx)).into_ref(&vm.ctx); - __init_dict(vm, &module); - extend_module(vm, &module).unwrap(); - module - } - }, ]); } - if !is_submodule && !context.has_extend_module { + if !is_submodule && !context.has_module_exec { items.push(parse_quote! { - pub(crate) fn extend_module(vm: &::rustpython_vm::VirtualMachine, module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>) -> ::rustpython_vm::PyResult<()> { - __extend_module(vm, module); + pub(crate) fn module_exec(vm: &::rustpython_vm::VirtualMachine, module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>) -> ::rustpython_vm::PyResult<()> { + __module_exec(vm, module); Ok(()) } }); @@ -192,11 +186,10 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result, ) { - module.__init_methods(vm).unwrap(); __init_attributes(vm, module); } }, diff --git a/crates/derive/src/lib.rs b/crates/derive/src/lib.rs index 5a3ff84c63a..1183b75b714 100644 --- a/crates/derive/src/lib.rs +++ b/crates/derive/src/lib.rs @@ -143,8 +143,8 @@ pub fn pyexception(attr: TokenStream, item: TokenStream) -> TokenStream { } /// This attribute must be applied to an inline module. -/// It defines a Python module in the form a `make_module` function in the module; -/// this has to be used in a `get_module_inits` to properly register the module. +/// It defines a Python module in the form of a `module_def` function in the module; +/// this has to be used in a `add_native_module` to properly register the module. /// Additionally, this macro defines 'MODULE_NAME' and 'DOC' in the module. /// # Arguments /// - `name`: the name of the python module, diff --git a/crates/stdlib/src/opcode.rs b/crates/stdlib/src/_opcode.rs similarity index 99% rename from crates/stdlib/src/opcode.rs rename to crates/stdlib/src/_opcode.rs index 2563dd7171c..408e63fb30e 100644 --- a/crates/stdlib/src/opcode.rs +++ b/crates/stdlib/src/_opcode.rs @@ -1,7 +1,7 @@ -pub(crate) use opcode::make_module; +pub(crate) use _opcode::module_def; #[pymodule] -mod opcode { +mod _opcode { use crate::vm::{ AsObject, PyObjectRef, PyResult, VirtualMachine, builtins::{PyInt, PyIntRef}, diff --git a/crates/stdlib/src/sqlite.rs b/crates/stdlib/src/_sqlite3.rs similarity index 99% rename from crates/stdlib/src/sqlite.rs rename to crates/stdlib/src/_sqlite3.rs index 9b7b810b25c..825fff06b75 100644 --- a/crates/stdlib/src/sqlite.rs +++ b/crates/stdlib/src/_sqlite3.rs @@ -8,18 +8,10 @@ // spell-checker:ignore cantlock commithook foreignkey notnull primarykey gettemppath autoindex convpath // spell-checker:ignore dbmoved vnode nbytes -use rustpython_vm::{AsObject, PyRef, VirtualMachine, builtins::PyModule}; - -// pub(crate) use _sqlite::make_module; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - // TODO: sqlite version check - let module = _sqlite::make_module(vm); - _sqlite::setup_module(module.as_object(), vm); - module -} +pub(crate) use _sqlite3::module_def; #[pymodule] -mod _sqlite { +mod _sqlite3 { use libsqlite3_sys::{ SQLITE_BLOB, SQLITE_DETERMINISTIC, SQLITE_FLOAT, SQLITE_INTEGER, SQLITE_NULL, SQLITE_OPEN_CREATE, SQLITE_OPEN_READWRITE, SQLITE_OPEN_URI, SQLITE_TEXT, SQLITE_TRACE_STMT, @@ -58,8 +50,8 @@ mod _sqlite { TryFromBorrowedObject, VirtualMachine, atomic_func, builtins::{ PyBaseException, PyBaseExceptionRef, PyByteArray, PyBytes, PyDict, PyDictRef, PyFloat, - PyInt, PyIntRef, PySlice, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, - PyUtf8Str, PyUtf8StrRef, + PyInt, PyIntRef, PyModule, PySlice, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, + PyTypeRef, PyUtf8Str, PyUtf8StrRef, }, convert::IntoObject, function::{ @@ -852,26 +844,26 @@ mod _sqlite { .expect("enable traceback not initialize") } - pub(super) fn setup_module(module: &PyObject, vm: &VirtualMachine) { + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + __module_exec(vm, module); + for (name, code) in ERROR_CODES { let name = vm.ctx.intern_str(*name); let code = vm.new_pyobj(*code); - module.set_attr(name, code, vm).unwrap(); + module.set_attr(name, code, vm)?; } - setup_module_exceptions(module, vm); + setup_module_exceptions(module.as_object(), vm); let _ = CONVERTERS.set(vm.ctx.new_dict()); let _ = ADAPTERS.set(vm.ctx.new_dict()); let _ = USER_FUNCTION_EXCEPTION.set(PyAtomicRef::from(None)); let _ = ENABLE_TRACEBACK.set(Radium::new(false)); - module - .set_attr("converters", converters().to_owned(), vm) - .unwrap(); - module - .set_attr("adapters", adapters().to_owned(), vm) - .unwrap(); + module.set_attr("converters", converters().to_owned(), vm)?; + module.set_attr("adapters", adapters().to_owned(), vm)?; + + Ok(()) } #[pyattr] diff --git a/crates/stdlib/src/array.rs b/crates/stdlib/src/array.rs index 463027d9baa..3f33f23d78d 100644 --- a/crates/stdlib/src/array.rs +++ b/crates/stdlib/src/array.rs @@ -1,33 +1,6 @@ // spell-checker:ignore typecode tofile tolist fromfile -use rustpython_vm::{PyRef, VirtualMachine, builtins::PyModule}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - let module = array::make_module(vm); - - let array = module - .get_attr("array", vm) - .expect("Expect array has array type."); - - let collections_abc = vm - .import("collections.abc", 0) - .expect("Expect collections exist."); - let abc = collections_abc - .get_attr("abc", vm) - .expect("Expect collections has abc submodule."); - let mutable_sequence = abc - .get_attr("MutableSequence", vm) - .expect("Expect collections.abc has MutableSequence type."); - - let register = &mutable_sequence - .get_attr("register", vm) - .expect("Expect collections.abc.MutableSequence has register method."); - register - .call((array,), vm) - .expect("Expect collections.abc.MutableSequence.register(array.array) not fail."); - - module -} +pub(crate) use array::module_def; #[pymodule(name = "array")] mod array { @@ -1658,4 +1631,25 @@ mod array { }; PyArray::from(array).into_ref_with_type(vm, cls) } + + // Register array.array as collections.abc.MutableSequence + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py, + ) -> PyResult<()> { + __module_exec(vm, module); + + let array_type = module + .get_attr("array", vm) + .expect("array module has array type"); + + // vm.import returns the top-level module, so we need to get abc submodule + let collections_abc = vm.import("collections.abc", 0)?; + let abc = collections_abc.get_attr("abc", vm)?; + let mutable_sequence = abc.get_attr("MutableSequence", vm)?; + let register = mutable_sequence.get_attr("register", vm)?; + register.call((array_type,), vm)?; + + Ok(()) + } } diff --git a/crates/stdlib/src/binascii.rs b/crates/stdlib/src/binascii.rs index 671d1d9e253..ee55d482e4c 100644 --- a/crates/stdlib/src/binascii.rs +++ b/crates/stdlib/src/binascii.rs @@ -1,7 +1,7 @@ // spell-checker:ignore hexlify unhexlify uuencodes CRCTAB rlecode rledecode pub(super) use decl::crc32; -pub(crate) use decl::make_module; +pub(crate) use decl::module_def; use rustpython_vm::{VirtualMachine, builtins::PyBaseExceptionRef, convert::ToPyException}; const PAD: u8 = 61u8; diff --git a/crates/stdlib/src/bisect.rs b/crates/stdlib/src/bisect.rs index e712d0275d2..69b6e8aee46 100644 --- a/crates/stdlib/src/bisect.rs +++ b/crates/stdlib/src/bisect.rs @@ -1,4 +1,4 @@ -pub(crate) use _bisect::make_module; +pub(crate) use _bisect::module_def; #[pymodule] mod _bisect { diff --git a/crates/stdlib/src/blake2.rs b/crates/stdlib/src/blake2.rs index 4209c966e86..5d2e991b929 100644 --- a/crates/stdlib/src/blake2.rs +++ b/crates/stdlib/src/blake2.rs @@ -1,6 +1,6 @@ // spell-checker:ignore usedforsecurity HASHXOF -pub(crate) use _blake2::make_module; +pub(crate) use _blake2::module_def; #[pymodule] mod _blake2 { diff --git a/crates/stdlib/src/bz2.rs b/crates/stdlib/src/bz2.rs index 93142e92a68..575f33c4b8f 100644 --- a/crates/stdlib/src/bz2.rs +++ b/crates/stdlib/src/bz2.rs @@ -1,6 +1,6 @@ // spell-checker:ignore compresslevel -pub(crate) use _bz2::make_module; +pub(crate) use _bz2::module_def; #[pymodule] mod _bz2 { diff --git a/crates/stdlib/src/cmath.rs b/crates/stdlib/src/cmath.rs index e7abb3bcad3..e7ea317d212 100644 --- a/crates/stdlib/src/cmath.rs +++ b/crates/stdlib/src/cmath.rs @@ -1,4 +1,4 @@ -pub(crate) use cmath::make_module; +pub(crate) use cmath::module_def; #[pymodule] mod cmath { diff --git a/crates/stdlib/src/contextvars.rs b/crates/stdlib/src/contextvars.rs index 5608b39d1e0..57864bf45fa 100644 --- a/crates/stdlib/src/contextvars.rs +++ b/crates/stdlib/src/contextvars.rs @@ -1,20 +1,9 @@ -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule, class::StaticType}; +pub(crate) use _contextvars::module_def; + +use crate::vm::PyRef; use _contextvars::PyContext; use core::cell::RefCell; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - let module = _contextvars::make_module(vm); - let token_type = module.get_attr("Token", vm).unwrap(); - token_type - .set_attr( - "MISSING", - _contextvars::ContextTokenMissing::static_type().to_owned(), - vm, - ) - .unwrap(); - module -} - thread_local! { // TODO: Vec doesn't seem to match copy behavior static CONTEXTS: RefCell>> = RefCell::default(); @@ -643,4 +632,17 @@ mod _contextvars { fn copy_context(vm: &VirtualMachine) -> PyContext { PyContext::current(vm).copy() } + + // Set Token.MISSING attribute + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py, + ) -> PyResult<()> { + __module_exec(vm, module); + + let token_type = module.get_attr("Token", vm)?; + token_type.set_attr("MISSING", ContextTokenMissing::static_type().to_owned(), vm)?; + + Ok(()) + } } diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs index 4f6cbd76828..7c89ebbba66 100644 --- a/crates/stdlib/src/csv.rs +++ b/crates/stdlib/src/csv.rs @@ -1,4 +1,4 @@ -pub(crate) use _csv::make_module; +pub(crate) use _csv::module_def; #[pymodule] mod _csv { diff --git a/crates/stdlib/src/faulthandler.rs b/crates/stdlib/src/faulthandler.rs index 35238367f9f..cb8d47fb686 100644 --- a/crates/stdlib/src/faulthandler.rs +++ b/crates/stdlib/src/faulthandler.rs @@ -1,4 +1,4 @@ -pub(crate) use decl::make_module; +pub(crate) use decl::module_def; #[allow(static_mut_refs)] // TODO: group code only with static mut refs #[pymodule(name = "faulthandler")] diff --git a/crates/stdlib/src/fcntl.rs b/crates/stdlib/src/fcntl.rs index 477c1f54210..407a2dfd6b3 100644 --- a/crates/stdlib/src/fcntl.rs +++ b/crates/stdlib/src/fcntl.rs @@ -1,6 +1,6 @@ // spell-checker:disable -pub(crate) use fcntl::make_module; +pub(crate) use fcntl::module_def; #[pymodule] mod fcntl { diff --git a/crates/stdlib/src/gc.rs b/crates/stdlib/src/gc.rs index 5fc96a302f7..9359119a603 100644 --- a/crates/stdlib/src/gc.rs +++ b/crates/stdlib/src/gc.rs @@ -1,4 +1,4 @@ -pub(crate) use gc::make_module; +pub(crate) use gc::module_def; #[pymodule] mod gc { diff --git a/crates/stdlib/src/grp.rs b/crates/stdlib/src/grp.rs index 9f7e4195509..c1a52eee62e 100644 --- a/crates/stdlib/src/grp.rs +++ b/crates/stdlib/src/grp.rs @@ -1,5 +1,5 @@ // spell-checker:disable -pub(crate) use grp::make_module; +pub(crate) use grp::module_def; #[pymodule] mod grp { diff --git a/crates/stdlib/src/hashlib.rs b/crates/stdlib/src/hashlib.rs index 261663c8564..c9902eff60a 100644 --- a/crates/stdlib/src/hashlib.rs +++ b/crates/stdlib/src/hashlib.rs @@ -1,6 +1,6 @@ // spell-checker:ignore usedforsecurity HASHXOF -pub(crate) use _hashlib::make_module; +pub(crate) use _hashlib::module_def; #[pymodule] pub mod _hashlib { diff --git a/crates/stdlib/src/json.rs b/crates/stdlib/src/json.rs index 4bdf4533533..3baeba629c8 100644 --- a/crates/stdlib/src/json.rs +++ b/crates/stdlib/src/json.rs @@ -1,4 +1,4 @@ -pub(crate) use _json::make_module; +pub(crate) use _json::module_def; mod machinery; #[pymodule] diff --git a/crates/stdlib/src/lib.rs b/crates/stdlib/src/lib.rs index dc9bad4bd2f..63063ad2920 100644 --- a/crates/stdlib/src/lib.rs +++ b/crates/stdlib/src/lib.rs @@ -38,10 +38,10 @@ mod json; #[cfg(not(any(target_os = "ios", target_arch = "wasm32")))] mod locale; +mod _opcode; mod math; #[cfg(any(unix, windows))] mod mmap; -mod opcode; mod pyexpat; mod pystruct; mod random; @@ -65,6 +65,11 @@ mod posixshmem; #[cfg(unix)] mod posixsubprocess; // libc is missing constants on redox +#[cfg(all( + feature = "sqlite", + not(any(target_os = "android", target_arch = "wasm32")) +))] +mod _sqlite3; #[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))] mod grp; #[cfg(windows)] @@ -75,11 +80,6 @@ mod resource; mod scproxy; #[cfg(any(unix, windows, target_os = "wasi"))] mod select; -#[cfg(all( - feature = "sqlite", - not(any(target_os = "android", target_arch = "wasm32")) -))] -mod sqlite; #[cfg(all(not(target_arch = "wasm32"), feature = "ssl-openssl"))] mod openssl; @@ -105,131 +105,87 @@ mod tkinter; use rustpython_common as common; use rustpython_vm as vm; -use crate::vm::{builtins, stdlib::StdlibInitFunc}; -use alloc::borrow::Cow; - -pub fn get_module_inits() -> impl Iterator, StdlibInitFunc)> { - macro_rules! modules { - { - $( - #[cfg($cfg:meta)] - { $( $key:expr => $val:expr),* $(,)? } - )* - } => {{ - [ - $( - $(#[cfg($cfg)] (Cow::<'static, str>::from($key), Box::new($val) as StdlibInitFunc),)* - )* - ] - .into_iter() - }}; - } - modules! { - #[cfg(all())] - { - "array" => array::make_module, - "binascii" => binascii::make_module, - "_bisect" => bisect::make_module, - "_bz2" => bz2::make_module, - "cmath" => cmath::make_module, - "_contextvars" => contextvars::make_module, - "_csv" => csv::make_module, - "faulthandler" => faulthandler::make_module, - "gc" => gc::make_module, - "_hashlib" => hashlib::make_module, - "_sha1" => sha1::make_module, - "_sha3" => sha3::make_module, - "_sha256" => sha256::make_module, - "_sha512" => sha512::make_module, - "_md5" => md5::make_module, - "_blake2" => blake2::make_module, - "_json" => json::make_module, - "math" => math::make_module, - "pyexpat" => pyexpat::make_module, - "_opcode" => opcode::make_module, - "_random" => random::make_module, - "_statistics" => statistics::make_module, - "_struct" => pystruct::make_module, - "unicodedata" => unicodedata::make_module, - "zlib" => zlib::make_module, - "_statistics" => statistics::make_module, - "_suggestions" => suggestions::make_module, - // crate::vm::sysmodule::sysconfigdata_name() => sysconfigdata::make_module, - } +use crate::vm::{Context, builtins}; + +/// Returns module definitions for multi-phase init modules. +/// These modules are added to sys.modules BEFORE their exec function runs, +/// allowing safe circular imports. +pub fn stdlib_module_defs(ctx: &Context) -> Vec<&'static builtins::PyModuleDef> { + vec![ + _opcode::module_def(ctx), + array::module_def(ctx), + binascii::module_def(ctx), + bisect::module_def(ctx), + blake2::module_def(ctx), + bz2::module_def(ctx), + cmath::module_def(ctx), + contextvars::module_def(ctx), + csv::module_def(ctx), + faulthandler::module_def(ctx), #[cfg(any(unix, target_os = "wasi"))] - { - "fcntl" => fcntl::make_module, - } - #[cfg(any(unix, windows, target_os = "wasi"))] - { - "select" => select::make_module, - } - #[cfg(not(target_arch = "wasm32"))] - { - "_multiprocessing" => multiprocessing::make_module, - "_socket" => socket::make_module, - } + fcntl::module_def(ctx), + gc::module_def(ctx), + #[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))] + grp::module_def(ctx), + hashlib::module_def(ctx), + json::module_def(ctx), + #[cfg(not(any(target_os = "ios", target_arch = "wasm32")))] + locale::module_def(ctx), #[cfg(not(any(target_os = "android", target_arch = "wasm32")))] - { - "_lzma" => lzma::make_module, - } - #[cfg(all(feature = "sqlite", not(any(target_os = "android", target_arch = "wasm32"))))] - { - "_sqlite3" => sqlite::make_module, - } - #[cfg(all(not(target_arch = "wasm32"), feature = "ssl-rustls"))] - { - "_ssl" => ssl::make_module, - } + lzma::module_def(ctx), + math::module_def(ctx), + md5::module_def(ctx), + #[cfg(any(unix, windows))] + mmap::module_def(ctx), + #[cfg(not(target_arch = "wasm32"))] + multiprocessing::module_def(ctx), #[cfg(all(not(target_arch = "wasm32"), feature = "ssl-openssl"))] - { - "_ssl" => openssl::make_module, - } + openssl::module_def(ctx), #[cfg(windows)] - { - "_overlapped" => overlapped::make_module, - } - // Unix-only + overlapped::module_def(ctx), #[cfg(unix)] - { - "_posixsubprocess" => posixsubprocess::make_module, - } + posixsubprocess::module_def(ctx), #[cfg(all(unix, not(target_os = "redox"), not(target_os = "android")))] - { - "_posixshmem" => posixshmem::make_module, - } - #[cfg(any(unix, windows))] - { - "mmap" => mmap::make_module, - } + posixshmem::module_def(ctx), + pyexpat::module_def(ctx), + pystruct::module_def(ctx), + random::module_def(ctx), #[cfg(all(unix, not(target_os = "redox")))] - { - "syslog" => syslog::make_module, - "resource" => resource::make_module, - } - #[cfg(all(unix, not(any(target_os = "ios", target_os = "redox"))))] - { - "termios" => termios::make_module, - } - #[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))] - { - "grp" => grp::make_module, - } + resource::module_def(ctx), #[cfg(target_os = "macos")] - { - "_scproxy" => scproxy::make_module, - } - #[cfg(not(any(target_os = "android", target_os = "ios", target_os = "windows", target_arch = "wasm32", target_os = "redox")))] - { - "_uuid" => uuid::make_module, - } - #[cfg(not(any(target_os = "ios", target_arch = "wasm32")))] - { - "_locale" => locale::make_module, - } + scproxy::module_def(ctx), + #[cfg(any(unix, windows, target_os = "wasi"))] + select::module_def(ctx), + sha1::module_def(ctx), + sha256::module_def(ctx), + sha3::module_def(ctx), + sha512::module_def(ctx), + #[cfg(not(target_arch = "wasm32"))] + socket::module_def(ctx), + #[cfg(all( + feature = "sqlite", + not(any(target_os = "android", target_arch = "wasm32")) + ))] + _sqlite3::module_def(ctx), + #[cfg(all(not(target_arch = "wasm32"), feature = "ssl-rustls"))] + ssl::module_def(ctx), + statistics::module_def(ctx), + suggestions::module_def(ctx), + #[cfg(all(unix, not(target_os = "redox")))] + syslog::module_def(ctx), + #[cfg(all(unix, not(any(target_os = "ios", target_os = "redox"))))] + termios::module_def(ctx), #[cfg(feature = "tkinter")] - { - "_tkinter" => tkinter::make_module, - } - } + tkinter::module_def(ctx), + unicodedata::module_def(ctx), + #[cfg(not(any( + target_os = "android", + target_os = "ios", + target_os = "windows", + target_arch = "wasm32", + target_os = "redox" + )))] + uuid::module_def(ctx), + zlib::module_def(ctx), + ] } diff --git a/crates/stdlib/src/locale.rs b/crates/stdlib/src/locale.rs index 51cf352590b..98db77d0d75 100644 --- a/crates/stdlib/src/locale.rs +++ b/crates/stdlib/src/locale.rs @@ -1,6 +1,6 @@ // spell-checker:ignore abday abmon yesexpr noexpr CRNCYSTR RADIXCHAR AMPM THOUSEP -pub(crate) use _locale::make_module; +pub(crate) use _locale::module_def; #[cfg(windows)] #[repr(C)] diff --git a/crates/stdlib/src/lzma.rs b/crates/stdlib/src/lzma.rs index b18ac3ee69a..0189c9d2ed0 100644 --- a/crates/stdlib/src/lzma.rs +++ b/crates/stdlib/src/lzma.rs @@ -1,6 +1,6 @@ // spell-checker:ignore ARMTHUMB -pub(crate) use _lzma::make_module; +pub(crate) use _lzma::module_def; #[pymodule] mod _lzma { diff --git a/crates/stdlib/src/math.rs b/crates/stdlib/src/math.rs index 1e014e49e24..238c7fd4928 100644 --- a/crates/stdlib/src/math.rs +++ b/crates/stdlib/src/math.rs @@ -1,4 +1,4 @@ -pub(crate) use math::make_module; +pub(crate) use math::module_def; use crate::vm::{VirtualMachine, builtins::PyBaseExceptionRef}; diff --git a/crates/stdlib/src/md5.rs b/crates/stdlib/src/md5.rs index dca48242bbd..11923b165ed 100644 --- a/crates/stdlib/src/md5.rs +++ b/crates/stdlib/src/md5.rs @@ -1,4 +1,4 @@ -pub(crate) use _md5::make_module; +pub(crate) use _md5::module_def; #[pymodule] mod _md5 { diff --git a/crates/stdlib/src/mmap.rs b/crates/stdlib/src/mmap.rs index d3895521096..625b6db5f6c 100644 --- a/crates/stdlib/src/mmap.rs +++ b/crates/stdlib/src/mmap.rs @@ -1,6 +1,6 @@ // spell-checker:disable //! mmap module -pub(crate) use mmap::make_module; +pub(crate) use mmap::module_def; #[pymodule] mod mmap { diff --git a/crates/stdlib/src/multiprocessing.rs b/crates/stdlib/src/multiprocessing.rs index bda13e6191a..f3a7831defe 100644 --- a/crates/stdlib/src/multiprocessing.rs +++ b/crates/stdlib/src/multiprocessing.rs @@ -1,4 +1,4 @@ -pub(crate) use _multiprocessing::make_module; +pub(crate) use _multiprocessing::module_def; #[cfg(windows)] #[pymodule] diff --git a/crates/stdlib/src/openssl.rs b/crates/stdlib/src/openssl.rs index df153459f62..0ec6d58e50c 100644 --- a/crates/stdlib/src/openssl.rs +++ b/crates/stdlib/src/openssl.rs @@ -23,32 +23,25 @@ cfg_if::cfg_if! { } } -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; +pub(crate) use _ssl::module_def; + use openssl_probe::ProbeResult; use std::sync::LazyLock; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - // if openssl is vendored, it doesn't know the locations - // of system certificates - cache the probe result now. - #[cfg(openssl_vendored)] - LazyLock::force(&PROBE); - _ssl::make_module(vm) -} - // define our own copy of ProbeResult so we can handle the vendor case // easily, without having to have a bunch of cfgs cfg_if::cfg_if! { if #[cfg(openssl_vendored)] { static PROBE: LazyLock = LazyLock::new(openssl_probe::probe); - fn probe() -> &'static ProbeResult { &PROBE } } else { - static EMPTY_PROBE: LazyLock = LazyLock::new(|| ProbeResult { cert_file: None, cert_dir: vec![] }); - fn probe() -> &'static ProbeResult { - &EMPTY_PROBE - } + static PROBE: LazyLock = LazyLock::new(|| ProbeResult { cert_file: None, cert_dir: vec![] }); } } +fn probe() -> &'static ProbeResult { + &PROBE +} + #[allow(non_upper_case_globals)] #[pymodule(with(cert::ssl_cert, ssl_error::ssl_error, ossl101, ossl111, windows))] mod _ssl { @@ -67,8 +60,8 @@ mod _ssl { vm::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{ - PyBaseException, PyBaseExceptionRef, PyBytesRef, PyListRef, PyStrRef, PyType, - PyWeak, + PyBaseException, PyBaseExceptionRef, PyBytesRef, PyListRef, PyModule, PyStrRef, + PyType, PyWeak, }, class_or_notimplemented, convert::ToPyException, @@ -104,6 +97,16 @@ mod _ssl { // Import certificate types from parent module use super::cert::{self, cert_to_certificate, cert_to_py}; + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + // if openssl is vendored, it doesn't know the locations + // of system certificates - cache the probe result now. + #[cfg(openssl_vendored)] + std::sync::LazyLock::force(&super::PROBE); + + __module_exec(vm, module); + Ok(()) + } + // Re-export PySSLCertificate to make it available in the _ssl module // It will be automatically exposed to Python via #[pyclass] #[allow(unused_imports)] diff --git a/crates/stdlib/src/overlapped.rs b/crates/stdlib/src/overlapped.rs index 664aa3f392c..b591a2f0f0f 100644 --- a/crates/stdlib/src/overlapped.rs +++ b/crates/stdlib/src/overlapped.rs @@ -1,6 +1,6 @@ // spell-checker:disable -pub(crate) use _overlapped::make_module; +pub(crate) use _overlapped::module_def; #[allow(non_snake_case)] #[pymodule] diff --git a/crates/stdlib/src/posixshmem.rs b/crates/stdlib/src/posixshmem.rs index a52866b7985..2a142d8b6f3 100644 --- a/crates/stdlib/src/posixshmem.rs +++ b/crates/stdlib/src/posixshmem.rs @@ -1,5 +1,5 @@ #[cfg(all(unix, not(target_os = "redox"), not(target_os = "android")))] -pub(crate) use _posixshmem::make_module; +pub(crate) use _posixshmem::module_def; #[cfg(all(unix, not(target_os = "redox"), not(target_os = "android")))] #[pymodule] diff --git a/crates/stdlib/src/posixsubprocess.rs b/crates/stdlib/src/posixsubprocess.rs index a6badc081d2..3d5e4c356d9 100644 --- a/crates/stdlib/src/posixsubprocess.rs +++ b/crates/stdlib/src/posixsubprocess.rs @@ -22,7 +22,7 @@ use alloc::ffi::CString; use core::{convert::Infallible as Never, ffi::CStr, marker::PhantomData, ops::Deref}; -pub(crate) use _posixsubprocess::make_module; +pub(crate) use _posixsubprocess::module_def; #[pymodule] mod _posixsubprocess { diff --git a/crates/stdlib/src/pyexpat.rs b/crates/stdlib/src/pyexpat.rs index 89cf690770d..4a33e032ef1 100644 --- a/crates/stdlib/src/pyexpat.rs +++ b/crates/stdlib/src/pyexpat.rs @@ -2,18 +2,7 @@ // spell-checker: ignore libexpat -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule, extend_module}; - -pub fn make_module(vm: &VirtualMachine) -> PyRef { - let module = _pyexpat::make_module(vm); - - extend_module!(vm, &module, { - "errors" => _errors::make_module(vm), - "model" => _model::make_module(vm), - }); - - module -} +pub(crate) use _pyexpat::module_def; macro_rules! create_property { ($ctx: expr, $attributes: expr, $name: expr, $class: expr, $element: ident) => { @@ -52,7 +41,8 @@ macro_rules! create_bool_property { mod _pyexpat { use crate::vm::{ Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, - builtins::{PyBytesRef, PyStr, PyStrRef, PyType}, + builtins::{PyBytesRef, PyModule, PyStr, PyStrRef, PyType}, + extend_module, function::ArgBytesLike, function::{Either, IntoFuncArgs, OptionalArg}, }; @@ -60,6 +50,21 @@ mod _pyexpat { use std::io::Cursor; use xml::reader::XmlEvent; + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + __module_exec(vm, module); + + // Add submodules + let model = super::_model::module_def(&vm.ctx).create_module(vm)?; + let errors = super::_errors::module_def(&vm.ctx).create_module(vm)?; + + extend_module!(vm, module, { + "model" => model, + "errors" => errors, + }); + + Ok(()) + } + type MutableObject = PyRwLock; #[pyattr(name = "version_info")] diff --git a/crates/stdlib/src/pystruct.rs b/crates/stdlib/src/pystruct.rs index 8801f0d705e..d3be417edb3 100644 --- a/crates/stdlib/src/pystruct.rs +++ b/crates/stdlib/src/pystruct.rs @@ -5,7 +5,7 @@ //! Use this rust module to do byte packing: //! -pub(crate) use _struct::make_module; +pub(crate) use _struct::module_def; #[pymodule] pub(crate) mod _struct { diff --git a/crates/stdlib/src/random.rs b/crates/stdlib/src/random.rs index be31d3011d7..35e6473d8f6 100644 --- a/crates/stdlib/src/random.rs +++ b/crates/stdlib/src/random.rs @@ -1,6 +1,6 @@ //! Random module. -pub(crate) use _random::make_module; +pub(crate) use _random::module_def; #[pymodule] mod _random { diff --git a/crates/stdlib/src/re.rs b/crates/stdlib/src/re.rs index 5af45671522..fdb14d427fc 100644 --- a/crates/stdlib/src/re.rs +++ b/crates/stdlib/src/re.rs @@ -1,4 +1,4 @@ -pub(crate) use re::make_module; +pub(crate) use re::module_def; #[pymodule] mod re { diff --git a/crates/stdlib/src/resource.rs b/crates/stdlib/src/resource.rs index e6df75e4b01..34c8161e0cd 100644 --- a/crates/stdlib/src/resource.rs +++ b/crates/stdlib/src/resource.rs @@ -1,6 +1,6 @@ // spell-checker:disable -pub(crate) use resource::make_module; +pub(crate) use resource::module_def; #[pymodule] mod resource { diff --git a/crates/stdlib/src/scproxy.rs b/crates/stdlib/src/scproxy.rs index b105dc4f2fb..09e7cdc6046 100644 --- a/crates/stdlib/src/scproxy.rs +++ b/crates/stdlib/src/scproxy.rs @@ -1,4 +1,4 @@ -pub(crate) use _scproxy::make_module; +pub(crate) use _scproxy::module_def; #[pymodule] mod _scproxy { diff --git a/crates/stdlib/src/select.rs b/crates/stdlib/src/select.rs index 3c2f5e63c7c..09a85cf1c00 100644 --- a/crates/stdlib/src/select.rs +++ b/crates/stdlib/src/select.rs @@ -1,25 +1,13 @@ // spell-checker:disable +pub(crate) use decl::module_def; + use crate::vm::{ - PyObject, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::PyListRef, - builtins::PyModule, + PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::PyListRef, }; use core::mem; use std::io; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - #[cfg(windows)] - crate::vm::windows::init_winsock(); - - #[cfg(unix)] - { - use crate::vm::class::PyClassImpl; - decl::poll::PyPoll::make_class(&vm.ctx); - } - - decl::make_module(vm) -} - #[cfg(unix)] mod platform { pub use libc::{FD_ISSET, FD_SET, FD_SETSIZE, FD_ZERO, fd_set, select, timeval}; @@ -221,13 +209,27 @@ fn sec_to_timeval(sec: f64) -> timeval { mod decl { use super::*; use crate::vm::{ - PyObjectRef, PyResult, VirtualMachine, - builtins::PyTypeRef, + Py, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyModule, PyTypeRef}, convert::ToPyException, function::{Either, OptionalOption}, stdlib::time, }; + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + #[cfg(windows)] + crate::vm::windows::init_winsock(); + + #[cfg(unix)] + { + use crate::vm::class::PyClassImpl; + poll::PyPoll::make_class(&vm.ctx); + } + + __module_exec(vm, module); + Ok(()) + } + #[pyattr] fn error(vm: &VirtualMachine) -> PyTypeRef { vm.ctx.exceptions.os_error.to_owned() @@ -545,7 +547,7 @@ mod decl { pub(super) mod epoll { use super::*; use crate::vm::{ - Py, PyPayload, + Py, PyPayload, PyRef, builtins::PyType, common::lock::{PyRwLock, PyRwLockReadGuard}, convert::{IntoPyException, ToPyObject}, diff --git a/crates/stdlib/src/sha1.rs b/crates/stdlib/src/sha1.rs index 04845bb76b5..28d4bcbbee1 100644 --- a/crates/stdlib/src/sha1.rs +++ b/crates/stdlib/src/sha1.rs @@ -1,4 +1,4 @@ -pub(crate) use _sha1::make_module; +pub(crate) use _sha1::module_def; #[pymodule] mod _sha1 { diff --git a/crates/stdlib/src/sha256.rs b/crates/stdlib/src/sha256.rs index 5d031968aeb..bf4a39373ea 100644 --- a/crates/stdlib/src/sha256.rs +++ b/crates/stdlib/src/sha256.rs @@ -1,14 +1,7 @@ -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - let _ = vm.import("_hashlib", 0); - _sha256::make_module(vm) -} - #[pymodule] mod _sha256 { use crate::hashlib::_hashlib::{HashArgs, local_sha224, local_sha256}; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; + use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; #[pyfunction] fn sha224(args: HashArgs, vm: &VirtualMachine) -> PyResult { @@ -19,4 +12,12 @@ mod _sha256 { fn sha256(args: HashArgs, vm: &VirtualMachine) -> PyResult { Ok(local_sha256(args).into_pyobject(vm)) } + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + let _ = vm.import("_hashlib", 0); + __module_exec(vm, module); + Ok(()) + } } + +pub(crate) use _sha256::module_def; diff --git a/crates/stdlib/src/sha3.rs b/crates/stdlib/src/sha3.rs index 07b61d9aed2..e7342d0dd69 100644 --- a/crates/stdlib/src/sha3.rs +++ b/crates/stdlib/src/sha3.rs @@ -1,4 +1,4 @@ -pub(crate) use _sha3::make_module; +pub(crate) use _sha3::module_def; #[pymodule] mod _sha3 { diff --git a/crates/stdlib/src/sha512.rs b/crates/stdlib/src/sha512.rs index baf63fdacf0..303c0067daa 100644 --- a/crates/stdlib/src/sha512.rs +++ b/crates/stdlib/src/sha512.rs @@ -1,14 +1,7 @@ -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - let _ = vm.import("_hashlib", 0); - _sha512::make_module(vm) -} - #[pymodule] mod _sha512 { use crate::hashlib::_hashlib::{HashArgs, local_sha384, local_sha512}; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; + use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; #[pyfunction] fn sha384(args: HashArgs, vm: &VirtualMachine) -> PyResult { @@ -19,4 +12,12 @@ mod _sha512 { fn sha512(args: HashArgs, vm: &VirtualMachine) -> PyResult { Ok(local_sha512(args).into_pyobject(vm)) } + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + let _ = vm.import("_hashlib", 0); + __module_exec(vm, module); + Ok(()) + } } + +pub(crate) use _sha512::module_def; diff --git a/crates/stdlib/src/socket.rs b/crates/stdlib/src/socket.rs index ea5f39cb40d..b0ebd681e02 100644 --- a/crates/stdlib/src/socket.rs +++ b/crates/stdlib/src/socket.rs @@ -1,27 +1,32 @@ // spell-checker:disable -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; +pub(crate) use _socket::module_def; + #[cfg(feature = "ssl")] pub(super) use _socket::{PySocket, SelectKind, sock_select, timeout_error_msg}; -pub fn make_module(vm: &VirtualMachine) -> PyRef { - #[cfg(windows)] - crate::vm::windows::init_winsock(); - _socket::make_module(vm) -} - #[pymodule] mod _socket { use crate::common::lock::{PyMappedRwLockReadGuard, PyRwLock, PyRwLockReadGuard}; use crate::vm::{ AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - builtins::{PyBaseExceptionRef, PyListRef, PyOSError, PyStrRef, PyTupleRef, PyTypeRef}, + builtins::{ + PyBaseExceptionRef, PyListRef, PyModule, PyOSError, PyStrRef, PyTupleRef, PyTypeRef, + }, common::os::ErrorExt, convert::{IntoPyException, ToPyObject, TryFromBorrowedObject, TryFromObject}, function::{ArgBytesLike, ArgMemoryBuffer, Either, FsPath, OptionalArg, OptionalOption}, types::{Constructor, DefaultConstructor, Initializer, Representable}, utils::ToCString, }; + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + #[cfg(windows)] + crate::vm::windows::init_winsock(); + + __module_exec(vm, module); + Ok(()) + } use core::{ mem::MaybeUninit, net::{Ipv4Addr, Ipv6Addr, SocketAddr}, diff --git a/crates/stdlib/src/ssl.rs b/crates/stdlib/src/ssl.rs index 959525d1f12..7d27e259cae 100644 --- a/crates/stdlib/src/ssl.rs +++ b/crates/stdlib/src/ssl.rs @@ -25,7 +25,7 @@ mod compat; // SSL exception types (shared with openssl backend) mod error; -pub(crate) use _ssl::make_module; +pub(crate) use _ssl::module_def; #[allow(non_snake_case)] #[allow(non_upper_case_globals)] diff --git a/crates/stdlib/src/statistics.rs b/crates/stdlib/src/statistics.rs index 8be2447ffd6..641d8864ed3 100644 --- a/crates/stdlib/src/statistics.rs +++ b/crates/stdlib/src/statistics.rs @@ -1,4 +1,4 @@ -pub(crate) use _statistics::make_module; +pub(crate) use _statistics::module_def; #[pymodule] mod _statistics { diff --git a/crates/stdlib/src/suggestions.rs b/crates/stdlib/src/suggestions.rs index e49e9dd4a4b..e0667dfb553 100644 --- a/crates/stdlib/src/suggestions.rs +++ b/crates/stdlib/src/suggestions.rs @@ -1,4 +1,4 @@ -pub(crate) use _suggestions::make_module; +pub(crate) use _suggestions::module_def; #[pymodule] mod _suggestions { diff --git a/crates/stdlib/src/syslog.rs b/crates/stdlib/src/syslog.rs index b52d1415692..8f446a8e161 100644 --- a/crates/stdlib/src/syslog.rs +++ b/crates/stdlib/src/syslog.rs @@ -1,6 +1,6 @@ // spell-checker:ignore logoption openlog setlogmask upto NDELAY ODELAY -pub(crate) use syslog::make_module; +pub(crate) use syslog::module_def; #[pymodule(name = "syslog")] mod syslog { diff --git a/crates/stdlib/src/termios.rs b/crates/stdlib/src/termios.rs index a9ae1375c6f..48dfae25a73 100644 --- a/crates/stdlib/src/termios.rs +++ b/crates/stdlib/src/termios.rs @@ -1,6 +1,6 @@ // spell-checker:disable -pub(crate) use self::termios::make_module; +pub(crate) use self::termios::module_def; #[pymodule] mod termios { diff --git a/crates/stdlib/src/tkinter.rs b/crates/stdlib/src/tkinter.rs index 6b7c4387d6c..b258002c129 100644 --- a/crates/stdlib/src/tkinter.rs +++ b/crates/stdlib/src/tkinter.rs @@ -1,6 +1,6 @@ // spell-checker:ignore createcommand -pub(crate) use self::_tkinter::make_module; +pub(crate) use self::_tkinter::module_def; #[pymodule] mod _tkinter { diff --git a/crates/stdlib/src/unicodedata.rs b/crates/stdlib/src/unicodedata.rs index 68d9a17e575..a859e39df10 100644 --- a/crates/stdlib/src/unicodedata.rs +++ b/crates/stdlib/src/unicodedata.rs @@ -4,35 +4,12 @@ // spell-checker:ignore nfkc unistr unidata +pub(crate) use unicodedata::module_def; + use crate::vm::{ - PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::PyModule, - builtins::PyStr, convert::TryFromBorrowedObject, + PyObject, PyResult, VirtualMachine, builtins::PyStr, convert::TryFromBorrowedObject, }; -pub fn make_module(vm: &VirtualMachine) -> PyRef { - let module = unicodedata::make_module(vm); - - let ucd: PyObjectRef = unicodedata::Ucd::new(unic_ucd_age::UNICODE_VERSION) - .into_ref(&vm.ctx) - .into(); - - for attr in [ - "category", - "lookup", - "name", - "bidirectional", - "east_asian_width", - "normalize", - "mirrored", - ] { - crate::vm::extend_module!(vm, &module, { - attr => ucd.get_attr(attr, vm).unwrap(), - }); - } - - module -} - enum NormalizeForm { Nfc, Nfkc, @@ -60,7 +37,8 @@ impl<'a> TryFromBorrowedObject<'a> for NormalizeForm { #[pymodule] mod unicodedata { use crate::vm::{ - PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::PyStrRef, + Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyModule, PyStrRef}, function::OptionalArg, }; use itertools::Itertools; @@ -73,6 +51,27 @@ mod unicodedata { use unic_ucd_category::GeneralCategory; use unicode_bidi_mirroring::is_mirroring; + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + __module_exec(vm, module); + + // Add UCD methods as module-level functions + let ucd: PyObjectRef = Ucd::new(UNICODE_VERSION).into_ref(&vm.ctx).into(); + + for attr in [ + "category", + "lookup", + "name", + "bidirectional", + "east_asian_width", + "normalize", + "mirrored", + ] { + module.set_attr(attr, ucd.get_attr(attr, vm)?, vm)?; + } + + Ok(()) + } + #[pyattr] #[pyclass(name = "UCD")] #[derive(Debug, PyPayload)] diff --git a/crates/stdlib/src/uuid.rs b/crates/stdlib/src/uuid.rs index 3f75db402c8..3bc01610d43 100644 --- a/crates/stdlib/src/uuid.rs +++ b/crates/stdlib/src/uuid.rs @@ -1,4 +1,4 @@ -pub(crate) use _uuid::make_module; +pub(crate) use _uuid::module_def; #[pymodule] mod _uuid { diff --git a/crates/stdlib/src/zlib.rs b/crates/stdlib/src/zlib.rs index 632543c5c64..40269f12bbf 100644 --- a/crates/stdlib/src/zlib.rs +++ b/crates/stdlib/src/zlib.rs @@ -1,6 +1,6 @@ // spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz chunker -pub(crate) use zlib::make_module; +pub(crate) use zlib::module_def; #[pymodule] mod zlib { diff --git a/crates/vm/src/builtins/mod.rs b/crates/vm/src/builtins/mod.rs index e9ca1f8b403..994a4f9c758 100644 --- a/crates/vm/src/builtins/mod.rs +++ b/crates/vm/src/builtins/mod.rs @@ -49,7 +49,7 @@ pub use mappingproxy::PyMappingProxy; pub(crate) mod memory; pub use memory::PyMemoryView; pub(crate) mod module; -pub use module::{PyModule, PyModuleDef}; +pub use module::{PyModule, PyModuleDef, PyModuleSlots}; pub(crate) mod namespace; pub use namespace::PyNamespace; pub(crate) mod object; diff --git a/crates/vm/src/builtins/module.rs b/crates/vm/src/builtins/module.rs index 27cc7135278..60ffb1487fa 100644 --- a/crates/vm/src/builtins/module.rs +++ b/crates/vm/src/builtins/module.rs @@ -41,6 +41,46 @@ impl core::fmt::Debug for PyModuleSlots { } } +impl PyModuleDef { + /// Create a module from this definition (Phase 1 of multi-phase init). + /// + /// This performs: + /// 1. Create module object (using create slot if provided) + /// 2. Initialize module dict from def + /// 3. Add methods to module + /// + /// Does NOT add to sys.modules or call exec slot. + pub fn create_module(&'static self, vm: &VirtualMachine) -> PyResult> { + use crate::PyPayload; + + // Create module (use create slot if provided, else default creation) + let module = if let Some(create) = self.slots.create { + // Custom module creation + let spec = vm.ctx.new_str(self.name.as_str()); + create(vm, spec.as_object(), self)? + } else { + // Default module creation + PyModule::from_def(self).into_ref(&vm.ctx) + }; + + // Initialize module dict and methods + PyModule::__init_dict_from_def(vm, &module); + module.__init_methods(vm)?; + + Ok(module) + } + + /// Execute the module's exec slot (Phase 2 of multi-phase init). + /// + /// Calls the exec slot if present. Returns Ok(()) if no exec slot. + pub fn exec_module(&'static self, vm: &VirtualMachine, module: &Py) -> PyResult<()> { + if let Some(exec) = self.slots.exec { + exec(vm, module)?; + } + Ok(()) + } +} + #[allow(clippy::new_without_default)] // avoid Default implementation #[pyclass(module = false, name = "module")] #[derive(Debug)] diff --git a/crates/vm/src/import.rs b/crates/vm/src/import.rs index 80ccbe30def..d1b77407ef8 100644 --- a/crates/vm/src/import.rs +++ b/crates/vm/src/import.rs @@ -87,16 +87,36 @@ pub fn import_frozen(vm: &VirtualMachine, module_name: &str) -> PyResult { } pub fn import_builtin(vm: &VirtualMachine, module_name: &str) -> PyResult { - let make_module_func = vm.state.module_inits.get(module_name).ok_or_else(|| { - vm.new_import_error( - format!("Cannot import builtin module {module_name}"), - vm.ctx.new_str(module_name), - ) - })?; - let module = make_module_func(vm); let sys_modules = vm.sys_module.get_attr("modules", vm)?; - sys_modules.set_item(module_name, module.as_object().to_owned(), vm)?; - Ok(module.into()) + + // Check if already in sys.modules (handles recursive imports) + if let Ok(module) = sys_modules.get_item(module_name, vm) { + return Ok(module); + } + + // Try multi-phase init first (preferred for modules that import other modules) + if let Some(&def) = vm.state.module_defs.get(module_name) { + // Phase 1: Create and initialize module + let module = def.create_module(vm)?; + + // Add to sys.modules BEFORE exec (critical for circular import handling) + sys_modules.set_item(module_name, module.clone().into(), vm)?; + + // Phase 2: Call exec slot (can safely import other modules now) + // If exec fails, remove the partially-initialized module from sys.modules + if let Err(e) = def.exec_module(vm, &module) { + let _ = sys_modules.del_item(module_name, vm); + return Err(e); + } + + return Ok(module.into()); + } + + // Module not found in module_defs + Err(vm.new_import_error( + format!("Cannot import builtin module {module_name}"), + vm.ctx.new_str(module_name), + )) } #[cfg(feature = "rustpython-compiler")] diff --git a/crates/vm/src/lib.rs b/crates/vm/src/lib.rs index 3f0eee278a2..aaa94ff52c1 100644 --- a/crates/vm/src/lib.rs +++ b/crates/vm/src/lib.rs @@ -99,7 +99,7 @@ pub use self::object::{ AsObject, Py, PyAtomicRef, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, PyWeakRef, }; -pub use self::vm::{Context, Interpreter, Settings, VirtualMachine}; +pub use self::vm::{Context, Interpreter, InterpreterBuilder, Settings, VirtualMachine}; pub use rustpython_common as common; pub use rustpython_compiler_core::{bytecode, frozen}; diff --git a/crates/vm/src/stdlib/_abc.rs b/crates/vm/src/stdlib/_abc.rs index ef528d95731..e04566b4b2d 100644 --- a/crates/vm/src/stdlib/_abc.rs +++ b/crates/vm/src/stdlib/_abc.rs @@ -3,7 +3,7 @@ //! This module provides the C implementation of Abstract Base Classes (ABCs) //! as defined in PEP 3119. -pub(crate) use _abc::make_module; +pub(crate) use _abc::module_def; #[pymodule] mod _abc { diff --git a/crates/vm/src/stdlib/ast.rs b/crates/vm/src/stdlib/ast.rs index 116d239c033..cb7bfc289b0 100644 --- a/crates/vm/src/stdlib/ast.rs +++ b/crates/vm/src/stdlib/ast.rs @@ -1,14 +1,16 @@ //! `ast` standard module for abstract syntax trees. + //! //! This module makes use of the parser logic, and translates all ast nodes //! into python ast.AST objects. +pub(crate) use python::_ast::module_def; + mod pyast; use crate::builtins::{PyInt, PyStr}; use crate::stdlib::ast::module::{Mod, ModInteractive}; use crate::stdlib::ast::node::BoxedSlice; -use crate::stdlib::ast::python::_ast; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, TryFromObject, VirtualMachine, @@ -365,9 +367,3 @@ pub const PY_COMPILE_FLAGS_MASK: i32 = PY_COMPILE_FLAG_AST_ONLY | CO_FUTURE_BARRY_AS_BDFL | CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS; - -pub fn make_module(vm: &VirtualMachine) -> PyRef { - let module = _ast::make_module(vm); - pyast::extend_module_nodes(vm, &module); - module -} diff --git a/crates/vm/src/stdlib/ast/python.rs b/crates/vm/src/stdlib/ast/python.rs index 35fe561527b..2ae07c34d07 100644 --- a/crates/vm/src/stdlib/ast/python.rs +++ b/crates/vm/src/stdlib/ast/python.rs @@ -96,4 +96,13 @@ pub(crate) mod _ast { #[pyattr(name = "PyCF_TYPE_COMMENTS")] use super::PY_CF_TYPE_COMMENTS; + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py, + ) -> PyResult<()> { + __module_exec(vm, module); + super::super::pyast::extend_module_nodes(vm, module); + Ok(()) + } } diff --git a/crates/vm/src/stdlib/atexit.rs b/crates/vm/src/stdlib/atexit.rs index 2286c36f1db..338fae3b2b7 100644 --- a/crates/vm/src/stdlib/atexit.rs +++ b/crates/vm/src/stdlib/atexit.rs @@ -1,5 +1,5 @@ pub use atexit::_run_exitfuncs; -pub(crate) use atexit::make_module; +pub(crate) use atexit::module_def; #[pymodule] mod atexit { diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index b4554bc30aa..1f14f6f5b04 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -2,7 +2,7 @@ //! //! Implements the list of [builtin Python functions](https://docs.python.org/3/library/builtins.html). use crate::{Py, VirtualMachine, builtins::PyModule, class::PyClassImpl}; -pub(crate) use builtins::{__module_def, DOC}; +pub(crate) use builtins::{DOC, module_def}; pub use builtins::{ascii, print, reversed}; #[pymodule] @@ -1100,7 +1100,8 @@ pub fn init_module(vm: &VirtualMachine, module: &Py) { crate::protocol::VecBuffer::make_class(&vm.ctx); - builtins::extend_module(vm, module).unwrap(); + module.__init_methods(vm).unwrap(); + builtins::module_exec(vm, module).unwrap(); let debug_mode: bool = vm.state.config.settings.optimize == 0; // Create dynamic ExceptionGroup with multiple inheritance (BaseExceptionGroup + Exception) diff --git a/crates/vm/src/stdlib/codecs.rs b/crates/vm/src/stdlib/codecs.rs index 1661eef1750..1b728386671 100644 --- a/crates/vm/src/stdlib/codecs.rs +++ b/crates/vm/src/stdlib/codecs.rs @@ -1,4 +1,4 @@ -pub(crate) use _codecs::make_module; +pub(crate) use _codecs::module_def; #[pymodule] mod _codecs { diff --git a/crates/vm/src/stdlib/collections.rs b/crates/vm/src/stdlib/collections.rs index fd48cea0598..0f84db80e74 100644 --- a/crates/vm/src/stdlib/collections.rs +++ b/crates/vm/src/stdlib/collections.rs @@ -1,4 +1,4 @@ -pub(crate) use _collections::make_module; +pub(crate) use _collections::module_def; #[pymodule] mod _collections { diff --git a/crates/vm/src/stdlib/ctypes.rs b/crates/vm/src/stdlib/ctypes.rs index cbe3f5405b6..e219bb4f985 100644 --- a/crates/vm/src/stdlib/ctypes.rs +++ b/crates/vm/src/stdlib/ctypes.rs @@ -10,8 +10,8 @@ mod structure; mod union; use crate::{ - AsObject, Py, PyObjectRef, PyRef, PyResult, VirtualMachine, - builtins::{PyModule, PyStr, PyType}, + AsObject, Py, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyStr, PyType}, class::PyClassImpl, types::TypeDataRef, }; @@ -85,32 +85,9 @@ impl PyType { } // Dynamic type check helpers for PyCData -// These check if an object's type's metaclass is a subclass of a specific metaclass +pub(crate) use _ctypes::module_def; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - let module = _ctypes::make_module(vm); - let ctx = &vm.ctx; - PyCSimpleType::make_class(ctx); - array::PyCArrayType::make_class(ctx); - pointer::PyCPointerType::make_class(ctx); - structure::PyCStructType::make_class(ctx); - union::PyCUnionType::make_class(ctx); - function::PyCFuncPtrType::make_class(ctx); - extend_module!(vm, &module, { - "_CData" => PyCData::make_class(ctx), - "_SimpleCData" => PyCSimple::make_class(ctx), - "Array" => PyCArray::make_class(ctx), - "CField" => PyCField::make_class(ctx), - "CFuncPtr" => function::PyCFuncPtr::make_class(ctx), - "_Pointer" => PyCPointer::make_class(ctx), - "_pointer_type_cache" => ctx.new_dict(), - "_array_type_cache" => ctx.new_dict(), - "Structure" => PyCStructure::make_class(ctx), - "CThunkObject" => function::PyCThunk::make_class(ctx), - "Union" => PyCUnion::make_class(ctx), - }); - module -} +// These check if an object's type's metaclass is a subclass of a specific metaclass /// Size of long double - platform dependent /// x86_64 macOS/Linux: 16 bytes (80-bit extended + padding) @@ -1300,4 +1277,37 @@ pub(crate) mod _ctypes { Ok(S_OK) } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py, + ) -> PyResult<()> { + use super::*; + + __module_exec(vm, module); + + let ctx = &vm.ctx; + PyCSimpleType::make_class(ctx); + array::PyCArrayType::make_class(ctx); + pointer::PyCPointerType::make_class(ctx); + structure::PyCStructType::make_class(ctx); + union::PyCUnionType::make_class(ctx); + function::PyCFuncPtrType::make_class(ctx); + + extend_module!(vm, module, { + "_CData" => PyCData::make_class(ctx), + "_SimpleCData" => PyCSimple::make_class(ctx), + "Array" => PyCArray::make_class(ctx), + "CField" => PyCField::make_class(ctx), + "CFuncPtr" => function::PyCFuncPtr::make_class(ctx), + "_Pointer" => PyCPointer::make_class(ctx), + "_pointer_type_cache" => ctx.new_dict(), + "_array_type_cache" => ctx.new_dict(), + "Structure" => PyCStructure::make_class(ctx), + "CThunkObject" => function::PyCThunk::make_class(ctx), + "Union" => PyCUnion::make_class(ctx), + }); + + Ok(()) + } } diff --git a/crates/vm/src/stdlib/errno.rs b/crates/vm/src/stdlib/errno.rs index 7a78ceaea83..8e4efbaafe9 100644 --- a/crates/vm/src/stdlib/errno.rs +++ b/crates/vm/src/stdlib/errno.rs @@ -1,25 +1,26 @@ // spell-checker:disable -use crate::{PyRef, VirtualMachine, builtins::PyModule}; +pub(crate) use errno_mod::module_def; -#[pymodule] -mod errno {} +#[pymodule(name = "errno")] +mod errno_mod { + use crate::{Py, PyResult, VirtualMachine, builtins::PyModule}; -pub fn make_module(vm: &VirtualMachine) -> PyRef { - let module = errno::make_module(vm); - let errorcode = vm.ctx.new_dict(); - extend_module!(vm, &module, { - "errorcode" => errorcode.clone(), - }); - for (name, code) in ERROR_CODES { - let name = vm.ctx.intern_str(*name); - let code = vm.new_pyobj(*code); - errorcode - .set_item(&*code, name.to_owned().into(), vm) - .unwrap(); - module.set_attr(name, code, vm).unwrap(); + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + __module_exec(vm, module); + + let errorcode = vm.ctx.new_dict(); + extend_module!(vm, module, { + "errorcode" => errorcode.clone(), + }); + for (name, code) in super::ERROR_CODES { + let name = vm.ctx.intern_str(*name); + let code = vm.new_pyobj(*code); + errorcode.set_item(&*code, name.to_owned().into(), vm)?; + module.set_attr(name, code, vm)?; + } + Ok(()) } - module } #[cfg(any(unix, windows, target_os = "wasi"))] diff --git a/crates/vm/src/stdlib/functools.rs b/crates/vm/src/stdlib/functools.rs index 307f7760f60..6c5c8f2e4c5 100644 --- a/crates/vm/src/stdlib/functools.rs +++ b/crates/vm/src/stdlib/functools.rs @@ -1,4 +1,4 @@ -pub(crate) use _functools::make_module; +pub(crate) use _functools::module_def; #[pymodule] mod _functools { diff --git a/crates/vm/src/stdlib/imp.rs b/crates/vm/src/stdlib/imp.rs index ad2cede975c..1c78e835a2d 100644 --- a/crates/vm/src/stdlib/imp.rs +++ b/crates/vm/src/stdlib/imp.rs @@ -1,6 +1,6 @@ use crate::frozen::FrozenModule; use crate::{VirtualMachine, builtins::PyBaseExceptionRef}; -pub(crate) use _imp::make_module; +pub(crate) use _imp::module_def; pub use crate::vm::resolve_frozen_alias; @@ -84,7 +84,7 @@ fn find_frozen(name: &str, vm: &VirtualMachine) -> Result bool { - vm.state.module_inits.contains_key(name.as_str()) + vm.state.module_defs.contains_key(name.as_str()) } #[pyfunction] @@ -119,19 +119,44 @@ mod _imp { let sys_modules = vm.sys_module.get_attr("modules", vm).unwrap(); let name: PyStrRef = spec.get_attr("name", vm)?.try_into_value(vm)?; - let module = if let Ok(module) = sys_modules.get_item(&*name, vm) { - module - } else if let Some(make_module_func) = vm.state.module_inits.get(name.as_str()) { - make_module_func(vm).into() - } else { - vm.ctx.none() - }; - Ok(module) + // Check sys.modules first + if let Ok(module) = sys_modules.get_item(&*name, vm) { + return Ok(module); + } + + // Try multi-phase init modules first (they need special handling) + if let Some(&def) = vm.state.module_defs.get(name.as_str()) { + // Phase 1: Create module (use create slot if provided, else default creation) + let module = if let Some(create) = def.slots.create { + // Custom module creation + create(vm, &spec, def)? + } else { + // Default module creation + PyModule::from_def(def).into_ref(&vm.ctx) + }; + + // Initialize module dict and methods + // Corresponds to PyModule_FromDefAndSpec: md_def, _add_methods_to_object, PyModule_SetDocString + PyModule::__init_dict_from_def(vm, &module); + module.__init_methods(vm)?; + + // Add to sys.modules BEFORE exec (critical for circular import handling) + sys_modules.set_item(&*name, module.clone().into(), vm)?; + + // Phase 2: Call exec slot (can safely import other modules now) + if let Some(exec) = def.slots.exec { + exec(vm, &module)?; + } + + return Ok(module.into()); + } + + Ok(vm.ctx.none()) } #[pyfunction] fn exec_builtin(_mod: PyRef) -> i32 { - // TODO: Should we do something here? + // For multi-phase init modules, exec is already called in create_builtin 0 } diff --git a/crates/vm/src/stdlib/io.rs b/crates/vm/src/stdlib/io.rs index 9cb0b7d7d2a..b98de6a87a5 100644 --- a/crates/vm/src/stdlib/io.rs +++ b/crates/vm/src/stdlib/io.rs @@ -1,6 +1,8 @@ /* * I/O core tools. */ +pub(crate) use _io::module_def; + cfg_if::cfg_if! { if #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] { use crate::common::crt_fd::Offset; @@ -19,7 +21,7 @@ cfg_if::cfg_if! { } use crate::{ - PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, + PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyModule}, common::os::ErrorExt, convert::{IntoPyException, ToPyException}, @@ -89,23 +91,6 @@ impl IntoPyException for std::io::Error { } } -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - let ctx = &vm.ctx; - - let module = _io::make_module(vm); - - #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] - fileio::extend_module(vm, &module).unwrap(); - - let unsupported_operation = _io::unsupported_operation().to_owned(); - extend_module!(vm, &module, { - "UnsupportedOperation" => unsupported_operation, - "BlockingIOError" => ctx.exceptions.blocking_io_error.to_owned(), - }); - - module -} - // not used on all platforms #[derive(Copy, Clone)] #[repr(transparent)] @@ -4743,8 +4728,23 @@ mod _io { assert_eq!(buffered.getvalue(), data); } } -} + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + // Call auto-generated initialization first + __module_exec(vm, module); + + // Initialize FileIO types on non-WASM platforms + #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] + super::fileio::module_exec(vm, module)?; + + let unsupported_operation = unsupported_operation().to_owned(); + extend_module!(vm, module, { + "UnsupportedOperation" => unsupported_operation, + "BlockingIOError" => vm.ctx.exceptions.blocking_io_error.to_owned(), + }); + Ok(()) + } +} // disable FileIO on WASM #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] #[pymodule] diff --git a/crates/vm/src/stdlib/itertools.rs b/crates/vm/src/stdlib/itertools.rs index f1761315cee..5a3cfee391b 100644 --- a/crates/vm/src/stdlib/itertools.rs +++ b/crates/vm/src/stdlib/itertools.rs @@ -1,4 +1,4 @@ -pub(crate) use decl::make_module; +pub(crate) use decl::module_def; #[pymodule(name = "itertools")] mod decl { diff --git a/crates/vm/src/stdlib/marshal.rs b/crates/vm/src/stdlib/marshal.rs index aced9e48773..cf7abe65194 100644 --- a/crates/vm/src/stdlib/marshal.rs +++ b/crates/vm/src/stdlib/marshal.rs @@ -1,5 +1,5 @@ // spell-checker:ignore pyfrozen pycomplex -pub(crate) use decl::make_module; +pub(crate) use decl::module_def; #[pymodule(name = "marshal")] mod decl { diff --git a/crates/vm/src/stdlib/mod.rs b/crates/vm/src/stdlib/mod.rs index 85c28983d74..088abc5b4a6 100644 --- a/crates/vm/src/stdlib/mod.rs +++ b/crates/vm/src/stdlib/mod.rs @@ -62,95 +62,66 @@ mod winapi; #[cfg(windows)] mod winreg; -use crate::{PyRef, VirtualMachine, builtins::PyModule}; -use alloc::borrow::Cow; -use std::collections::HashMap; +use crate::{Context, builtins::PyModuleDef}; -pub type StdlibInitFunc = Box PyRef)>; -pub type StdlibMap = HashMap, StdlibInitFunc, ahash::RandomState>; - -pub fn get_module_inits() -> StdlibMap { - macro_rules! modules { - { - $( - #[cfg($cfg:meta)] - { $( $key:expr => $val:expr),* $(,)? } - )* - } => {{ - let modules = [ - $( - $(#[cfg($cfg)] (Cow::<'static, str>::from($key), Box::new($val) as StdlibInitFunc),)* - )* - ]; - modules.into_iter().collect() - }}; - } - modules! { - #[cfg(all())] - { - "_abc" => _abc::make_module, - "atexit" => atexit::make_module, - "_codecs" => codecs::make_module, - "_collections" => collections::make_module, - "errno" => errno::make_module, - "_functools" => functools::make_module, - "itertools" => itertools::make_module, - "_io" => io::make_module, - "marshal" => marshal::make_module, - "_operator" => operator::make_module, - "_signal" => signal::make_module, - "_sre" => sre::make_module, - "_stat" => stat::make_module, - "_sysconfig" => sysconfig::make_module, - "_string" => string::make_module, - "time" => time::make_module, - "_typing" => typing::make_module, - "_weakref" => weakref::make_module, - "_imp" => imp::make_module, - "_warnings" => warnings::make_module, - sys::sysconfigdata_name() => sysconfigdata::make_module, - } - // parser related modules: +/// Returns module definitions for multi-phase init modules. +/// +/// These modules use multi-phase initialization pattern: +/// 1. Create module from def and add to sys.modules +/// 2. Call exec slot (can safely import other modules without circular import issues) +pub fn builtin_module_defs(ctx: &Context) -> Vec<&'static PyModuleDef> { + vec![ + _abc::module_def(ctx), #[cfg(feature = "ast")] - { - "_ast" => ast::make_module, - } - // compiler related modules: - #[cfg(feature = "compiler")] - { - "_symtable" => symtable::make_module, - } + ast::module_def(ctx), + atexit::module_def(ctx), + codecs::module_def(ctx), + collections::module_def(ctx), + #[cfg(all( + any(target_os = "linux", target_os = "macos", target_os = "windows"), + not(any(target_env = "musl", target_env = "sgx")) + ))] + ctypes::module_def(ctx), + errno::module_def(ctx), + functools::module_def(ctx), + imp::module_def(ctx), + io::module_def(ctx), + itertools::module_def(ctx), + marshal::module_def(ctx), + #[cfg(windows)] + msvcrt::module_def(ctx), + #[cfg(windows)] + nt::module_def(ctx), + operator::module_def(ctx), #[cfg(any(unix, target_os = "wasi"))] - { - "posix" => posix::make_module, - // "fcntl" => fcntl::make_module, - } - #[cfg(feature = "threading")] - { - "_thread" => thread::make_module, - } - // Unix-only + posix::module_def(ctx), + #[cfg(all( + any(not(target_arch = "wasm32"), target_os = "wasi"), + not(any(unix, windows)) + ))] + posix::module_def(ctx), #[cfg(all( unix, not(any(target_os = "ios", target_os = "wasi", target_os = "redox")) ))] - { - "pwd" => pwd::make_module, - } - // Windows-only + pwd::module_def(ctx), + signal::module_def(ctx), + sre::module_def(ctx), + stat::module_def(ctx), + string::module_def(ctx), + #[cfg(feature = "compiler")] + symtable::module_def(ctx), + sysconfigdata::module_def(ctx), + sysconfig::module_def(ctx), + #[cfg(feature = "threading")] + thread::module_def(ctx), + time::module_def(ctx), + typing::module_def(ctx), + warnings::module_def(ctx), + weakref::module_def(ctx), #[cfg(windows)] - { - "nt" => nt::make_module, - "msvcrt" => msvcrt::make_module, - "_winapi" => winapi::make_module, - "winreg" => winreg::make_module, - } - #[cfg(all( - any(target_os = "linux", target_os = "macos", target_os = "windows"), - not(any(target_env = "musl", target_env = "sgx")) - ))] - { - "_ctypes" => ctypes::make_module, - } - } + winapi::module_def(ctx), + #[cfg(windows)] + winreg::module_def(ctx), + ] } diff --git a/crates/vm/src/stdlib/nt.rs b/crates/vm/src/stdlib/nt.rs index a32959808c0..c8c699fb3fa 100644 --- a/crates/vm/src/stdlib/nt.rs +++ b/crates/vm/src/stdlib/nt.rs @@ -1,19 +1,12 @@ // spell-checker:disable -use crate::{PyRef, VirtualMachine, builtins::PyModule}; - +pub(crate) use module::module_def; pub use module::raw_set_handle_inheritable; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - let module = module::make_module(vm); - super::os::extend_module(vm, &module); - module -} - #[pymodule(name = "nt", with(super::os::_os))] pub(crate) mod module { use crate::{ - PyResult, TryFromObject, VirtualMachine, + Py, PyResult, TryFromObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyDictRef, PyListRef, PyStrRef, PyTupleRef}, common::{crt_fd, suppress_iph, windows::ToWideString}, convert::ToPyException, @@ -1923,4 +1916,13 @@ pub(crate) mod module { pub(crate) fn support_funcs() -> Vec { Vec::new() } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py, + ) -> PyResult<()> { + __module_exec(vm, module); + super::super::os::module_exec(vm, module)?; + Ok(()) + } } diff --git a/crates/vm/src/stdlib/operator.rs b/crates/vm/src/stdlib/operator.rs index 7877ddb0114..c5fa03a12d6 100644 --- a/crates/vm/src/stdlib/operator.rs +++ b/crates/vm/src/stdlib/operator.rs @@ -1,4 +1,4 @@ -pub(crate) use _operator::make_module; +pub(crate) use _operator::module_def; #[pymodule] mod _operator { diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index 00f2aea1dbd..691cb068605 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -149,6 +149,8 @@ pub(super) mod _os { use super::{DirFd, FollowSymlinks, SupportFunc}; #[cfg(windows)] use crate::common::windows::ToWideString; + #[cfg(any(unix, windows))] + use crate::utils::ToCString; use crate::{ AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, builtins::{ @@ -168,7 +170,6 @@ pub(super) mod _os { protocol::PyIterReturn, recursion::ReprGuard, types::{IterNext, Iterable, PyStructSequence, Representable, SelfIter}, - utils::ToCString, vm::VirtualMachine, }; use core::time::Duration; @@ -906,9 +907,11 @@ pub(super) mod _os { #[pyarg(any, default)] #[pystruct_sequence(skip)] pub st_blocks: i64, + #[cfg(windows)] #[pyarg(any, default)] #[pystruct_sequence(skip)] pub st_reparse_tag: u32, + #[cfg(windows)] #[pyarg(any, default)] #[pystruct_sequence(skip)] pub st_file_attributes: u32, @@ -943,13 +946,8 @@ pub(super) mod _os { #[cfg(windows)] let st_reparse_tag = stat.st_reparse_tag; - #[cfg(not(windows))] - let st_reparse_tag = 0; - #[cfg(windows)] let st_file_attributes = stat.st_file_attributes; - #[cfg(not(windows))] - let st_file_attributes = 0; // On Windows, combine st_ino and st_ino_high into a 128-bit value // like _pystat_l128_from_l64_l64 @@ -986,7 +984,9 @@ pub(super) mod _os { st_blksize, #[cfg(not(windows))] st_blocks, + #[cfg(windows)] st_reparse_tag, + #[cfg(windows)] st_file_attributes, } } @@ -1897,21 +1897,21 @@ impl SupportFunc { } } -pub fn extend_module(vm: &VirtualMachine, module: &Py) { +pub fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { let support_funcs = _os::support_funcs(); let supports_fd = PySet::default().into_ref(&vm.ctx); let supports_dir_fd = PySet::default().into_ref(&vm.ctx); let supports_follow_symlinks = PySet::default().into_ref(&vm.ctx); for support in support_funcs { - let func_obj = module.get_attr(support.name, vm).unwrap(); + let func_obj = module.get_attr(support.name, vm)?; if support.fd.unwrap_or(false) { - supports_fd.clone().add(func_obj.clone(), vm).unwrap(); + supports_fd.clone().add(func_obj.clone(), vm)?; } if support.dir_fd.unwrap_or(false) { - supports_dir_fd.clone().add(func_obj.clone(), vm).unwrap(); + supports_dir_fd.clone().add(func_obj.clone(), vm)?; } if support.follow_symlinks.unwrap_or(false) { - supports_follow_symlinks.clone().add(func_obj, vm).unwrap(); + supports_follow_symlinks.clone().add(func_obj, vm)?; } } @@ -1921,6 +1921,8 @@ pub fn extend_module(vm: &VirtualMachine, module: &Py) { "supports_follow_symlinks" => supports_follow_symlinks, "error" => vm.ctx.exceptions.os_error.to_owned(), }); + + Ok(()) } #[cfg(not(windows))] diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index 7ac5129d805..699989be133 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -1,8 +1,9 @@ // spell-checker:disable -use crate::{PyRef, VirtualMachine, builtins::PyModule}; use std::os::fd::BorrowedFd; +pub(crate) use module::module_def; + pub fn set_inheritable(fd: BorrowedFd<'_>, inheritable: bool) -> nix::Result<()> { use nix::fcntl; let flags = fcntl::FdFlag::from_bits_truncate(fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFD)?); @@ -14,12 +15,6 @@ pub fn set_inheritable(fd: BorrowedFd<'_>, inheritable: bool) -> nix::Result<()> Ok(()) } -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - let module = module::make_module(vm); - super::os::extend_module(vm, &module); - module -} - #[pymodule(name = "posix", with(super::os::_os))] pub mod module { use crate::{ @@ -2606,4 +2601,13 @@ pub mod module { } Ok(buf) } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py, + ) -> PyResult<()> { + __module_exec(vm, module); + super::super::os::module_exec(vm, module)?; + Ok(()) + } } diff --git a/crates/vm/src/stdlib/posix_compat.rs b/crates/vm/src/stdlib/posix_compat.rs index 0fabd6a1f45..89d3d94d7b2 100644 --- a/crates/vm/src/stdlib/posix_compat.rs +++ b/crates/vm/src/stdlib/posix_compat.rs @@ -1,18 +1,13 @@ // spell-checker:disable //! `posix` compatible module for `not(any(unix, windows))` -use crate::{PyRef, VirtualMachine, builtins::PyModule}; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - let module = module::make_module(vm); - super::os::extend_module(vm, &module); - module -} +pub(crate) use module::module_def; #[pymodule(name = "posix", with(super::os::_os))] pub(crate) mod module { use crate::{ - PyObjectRef, PyResult, VirtualMachine, + Py, PyObjectRef, PyResult, VirtualMachine, builtins::PyStrRef, convert::IntoPyException, ospath::OsPath, @@ -71,4 +66,13 @@ pub(crate) mod module { pub(crate) fn support_funcs() -> Vec { Vec::new() } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py, + ) -> PyResult<()> { + __module_exec(vm, module); + super::super::os::module_exec(vm, module)?; + Ok(()) + } } diff --git a/crates/vm/src/stdlib/pwd.rs b/crates/vm/src/stdlib/pwd.rs index 6405ed7be91..e8aee608cc7 100644 --- a/crates/vm/src/stdlib/pwd.rs +++ b/crates/vm/src/stdlib/pwd.rs @@ -1,6 +1,6 @@ // spell-checker:disable -pub(crate) use pwd::make_module; +pub(crate) use pwd::module_def; #[pymodule] mod pwd { diff --git a/crates/vm/src/stdlib/signal.rs b/crates/vm/src/stdlib/signal.rs index b316ea306d0..0426bab6c10 100644 --- a/crates/vm/src/stdlib/signal.rs +++ b/crates/vm/src/stdlib/signal.rs @@ -1,24 +1,12 @@ // spell-checker:disable -use crate::{PyRef, VirtualMachine, builtins::PyModule}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - let module = _signal::make_module(vm); - - #[cfg(any(unix, windows))] - _signal::init_signal_handlers(&module, vm); - - module -} +pub(crate) use _signal::module_def; #[pymodule] pub(crate) mod _signal { #[cfg(any(unix, windows))] - use crate::{ - Py, - convert::{IntoPyException, TryFromBorrowedObject}, - }; - use crate::{PyObjectRef, PyResult, VirtualMachine, signal}; + use crate::convert::{IntoPyException, TryFromBorrowedObject}; + use crate::{Py, PyObjectRef, PyResult, VirtualMachine, signal}; #[cfg(unix)] use crate::{ builtins::PyTypeRef, @@ -682,4 +670,16 @@ pub(crate) mod _signal { // TODO: handle _res < 1, support warn_on_full_buffer } } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py, + ) -> PyResult<()> { + __module_exec(vm, module); + + #[cfg(any(unix, windows))] + init_signal_handlers(module, vm); + + Ok(()) + } } diff --git a/crates/vm/src/stdlib/sre.rs b/crates/vm/src/stdlib/sre.rs index 04ff0695b6b..692b9754214 100644 --- a/crates/vm/src/stdlib/sre.rs +++ b/crates/vm/src/stdlib/sre.rs @@ -1,4 +1,4 @@ -pub(crate) use _sre::make_module; +pub(crate) use _sre::module_def; #[pymodule] mod _sre { diff --git a/crates/vm/src/stdlib/stat.rs b/crates/vm/src/stdlib/stat.rs index 90f045fcaf0..44b55628d6f 100644 --- a/crates/vm/src/stdlib/stat.rs +++ b/crates/vm/src/stdlib/stat.rs @@ -1,7 +1,7 @@ -use crate::{PyRef, VirtualMachine, builtins::PyModule}; +pub(crate) use _stat::module_def; #[pymodule] -mod stat { +mod _stat { // Use libc::mode_t for Mode to match the system's definition #[cfg(unix)] type Mode = libc::mode_t; @@ -522,7 +522,3 @@ mod stat { result } } - -pub fn make_module(vm: &VirtualMachine) -> PyRef { - stat::make_module(vm) -} diff --git a/crates/vm/src/stdlib/string.rs b/crates/vm/src/stdlib/string.rs index a9911f3d45f..c620f2a40e2 100644 --- a/crates/vm/src/stdlib/string.rs +++ b/crates/vm/src/stdlib/string.rs @@ -1,7 +1,7 @@ /* String builtin module */ -pub(crate) use _string::make_module; +pub(crate) use _string::module_def; #[pymodule] mod _string { diff --git a/crates/vm/src/stdlib/symtable.rs b/crates/vm/src/stdlib/symtable.rs index 51c5c8e47ea..881c5519843 100644 --- a/crates/vm/src/stdlib/symtable.rs +++ b/crates/vm/src/stdlib/symtable.rs @@ -1,7 +1,7 @@ -pub(crate) use symtable::make_module; +pub(crate) use _symtable::module_def; #[pymodule] -mod symtable { +mod _symtable { use crate::{ PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyDictRef, PyStrRef}, diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 665b0f1f495..9c0f84650bc 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -1,8 +1,6 @@ use crate::{Py, PyResult, VirtualMachine, builtins::PyModule, convert::ToPyObject}; -pub(crate) use sys::{ - __module_def, DOC, MAXSIZE, RUST_MULTIARCH, UnraisableHookArgsData, multiarch, -}; +pub(crate) use sys::{DOC, MAXSIZE, RUST_MULTIARCH, UnraisableHookArgsData, module_def, multiarch}; #[pymodule(name = "_jit")] mod sys_jit { @@ -80,7 +78,7 @@ mod sys { #[pyattr(name = "abiflags")] const ABIFLAGS_ATTR: &str = "t"; // 't' for free-threaded (no GIL) // Internal constant used for sysconfigdata_name - pub(crate) const ABIFLAGS: &str = "t"; + pub const ABIFLAGS: &str = "t"; #[pyattr(name = "api_version")] const API_VERSION: u32 = 0x0; // what C api? #[pyattr(name = "copyright")] @@ -96,7 +94,7 @@ mod sys { #[pyattr(name = "maxunicode")] const MAXUNICODE: u32 = core::char::MAX as u32; #[pyattr(name = "platform")] - pub(crate) const PLATFORM: &str = { + pub const PLATFORM: &str = { cfg_if::cfg_if! { if #[cfg(target_os = "linux")] { "linux" @@ -168,9 +166,11 @@ mod sys { #[pyattr] fn builtin_module_names(vm: &VirtualMachine) -> PyTupleRef { - let mut module_names: Vec<_> = vm.state.module_inits.keys().cloned().collect(); - module_names.push("sys".into()); - module_names.push("builtins".into()); + let mut module_names: Vec = + vm.state.module_defs.keys().map(|&s| s.to_owned()).collect(); + module_names.push("sys".to_owned()); + module_names.push("builtins".to_owned()); + module_names.sort(); vm.ctx.new_tuple( module_names @@ -1609,7 +1609,8 @@ mod sys { } pub(crate) fn init_module(vm: &VirtualMachine, module: &Py, builtins: &Py) { - sys::extend_module(vm, module).unwrap(); + module.__init_methods(vm).unwrap(); + sys::module_exec(vm, module).unwrap(); let modules = vm.ctx.new_dict(); modules @@ -1620,7 +1621,8 @@ pub(crate) fn init_module(vm: &VirtualMachine, module: &Py, builtins: .unwrap(); // Create sys._jit submodule - let jit_module = sys_jit::make_module(vm); + let jit_def = sys_jit::module_def(&vm.ctx); + let jit_module = jit_def.create_module(vm).unwrap(); extend_module!(vm, module, { "__doc__" => sys::DOC.to_owned().to_pyobject(vm), diff --git a/crates/vm/src/stdlib/sysconfig.rs b/crates/vm/src/stdlib/sysconfig.rs index df5b7100a90..724e1bcf979 100644 --- a/crates/vm/src/stdlib/sysconfig.rs +++ b/crates/vm/src/stdlib/sysconfig.rs @@ -1,7 +1,7 @@ -pub(crate) use sysconfig::make_module; +pub(crate) use _sysconfig::module_def; -#[pymodule(name = "_sysconfig")] -pub(crate) mod sysconfig { +#[pymodule] +pub(crate) mod _sysconfig { use crate::{VirtualMachine, builtins::PyDictRef, convert::ToPyObject}; #[pyfunction] diff --git a/crates/vm/src/stdlib/sysconfigdata.rs b/crates/vm/src/stdlib/sysconfigdata.rs index 99228024d2d..a10745a8cad 100644 --- a/crates/vm/src/stdlib/sysconfigdata.rs +++ b/crates/vm/src/stdlib/sysconfigdata.rs @@ -1,11 +1,29 @@ // spell-checker: words LDSHARED ARFLAGS CPPFLAGS CCSHARED BASECFLAGS BLDSHARED -pub(crate) use _sysconfigdata::make_module; +pub(crate) use _sysconfigdata::module_def; #[pymodule] -pub(crate) mod _sysconfigdata { - use crate::stdlib::sys::{RUST_MULTIARCH, multiarch}; - use crate::{VirtualMachine, builtins::PyDictRef, convert::ToPyObject}; +mod _sysconfigdata { + use crate::stdlib::sys::{RUST_MULTIARCH, multiarch, sysconfigdata_name}; + use crate::{ + Py, PyResult, VirtualMachine, + builtins::{PyDictRef, PyModule}, + convert::ToPyObject, + }; + + fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + // Set build_time_vars attribute + let build_time_vars = build_time_vars(vm); + module.set_attr("build_time_vars", build_time_vars, vm)?; + + // Ensure the module is registered under the platform-specific name + // (import_builtin() already handles this, but double-check for safety) + let sys_modules = vm.sys_module.get_attr("modules", vm)?; + let sysconfigdata_name = sysconfigdata_name(); + sys_modules.set_item(sysconfigdata_name.as_str(), module.to_owned().into(), vm)?; + + Ok(()) + } #[pyattr] fn build_time_vars(vm: &VirtualMachine) -> PyDictRef { diff --git a/crates/vm/src/stdlib/thread.rs b/crates/vm/src/stdlib/thread.rs index f7e47b15deb..650989c67b7 100644 --- a/crates/vm/src/stdlib/thread.rs +++ b/crates/vm/src/stdlib/thread.rs @@ -4,7 +4,7 @@ pub(crate) use _thread::after_fork_child; #[cfg_attr(target_arch = "wasm32", allow(unused_imports))] pub(crate) use _thread::{ CurrentFrameSlot, HandleEntry, RawRMutex, ShutdownEntry, get_all_current_frames, get_ident, - init_main_thread_ident, make_module, + init_main_thread_ident, module_def, }; #[pymodule] diff --git a/crates/vm/src/stdlib/time.rs b/crates/vm/src/stdlib/time.rs index 8539e69bacb..ccd80402bf7 100644 --- a/crates/vm/src/stdlib/time.rs +++ b/crates/vm/src/stdlib/time.rs @@ -3,18 +3,10 @@ // See also: // https://docs.python.org/3/library/time.html -use crate::{PyRef, VirtualMachine, builtins::PyModule}; pub use decl::time; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - #[cfg(not(target_env = "msvc"))] - #[cfg(not(target_arch = "wasm32"))] - unsafe { - c_tzset() - }; - decl::make_module(vm) -} +pub(crate) use decl::module_def; #[cfg(not(target_env = "msvc"))] #[cfg(not(target_arch = "wasm32"))] @@ -34,7 +26,7 @@ unsafe extern "C" { #[pymodule(name = "time", with(platform))] mod decl { use crate::{ - AsObject, PyObjectRef, PyResult, VirtualMachine, + AsObject, Py, PyObjectRef, PyResult, VirtualMachine, builtins::{PyStrRef, PyTypeRef, PyUtf8StrRef}, function::{Either, FuncArgs, OptionalArg}, types::{PyStructSequence, struct_sequence_new}, @@ -581,6 +573,20 @@ mod decl { #[allow(unused_imports)] use super::platform::*; + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py, + ) -> PyResult<()> { + #[cfg(not(target_env = "msvc"))] + #[cfg(not(target_arch = "wasm32"))] + unsafe { + super::c_tzset() + }; + + __module_exec(vm, module); + Ok(()) + } } #[cfg(unix)] diff --git a/crates/vm/src/stdlib/typing.rs b/crates/vm/src/stdlib/typing.rs index b8048b16d94..7cc5696f923 100644 --- a/crates/vm/src/stdlib/typing.rs +++ b/crates/vm/src/stdlib/typing.rs @@ -1,10 +1,11 @@ // spell-checker:ignore typevarobject funcobj -use crate::{Context, PyPayload, PyRef, VirtualMachine, class::PyClassImpl, stdlib::PyModule}; +use crate::{Context, class::PyClassImpl}; pub use crate::stdlib::typevar::{ Generic, ParamSpec, ParamSpecArgs, ParamSpecKwargs, TypeVar, TypeVarTuple, set_typeparam_default, }; +pub(crate) use decl::module_def; pub use decl::*; /// Initialize typing types (call extend_class) @@ -12,32 +13,12 @@ pub fn init(ctx: &Context) { NoDefault::extend_class(ctx, ctx.types.typing_no_default_type); } -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - let module = decl::make_module(vm); - TypeVar::make_class(&vm.ctx); - ParamSpec::make_class(&vm.ctx); - TypeVarTuple::make_class(&vm.ctx); - ParamSpecArgs::make_class(&vm.ctx); - ParamSpecKwargs::make_class(&vm.ctx); - Generic::make_class(&vm.ctx); - extend_module!(vm, &module, { - "NoDefault" => vm.ctx.typing_no_default.clone(), - "TypeVar" => TypeVar::class(&vm.ctx).to_owned(), - "ParamSpec" => ParamSpec::class(&vm.ctx).to_owned(), - "TypeVarTuple" => TypeVarTuple::class(&vm.ctx).to_owned(), - "ParamSpecArgs" => ParamSpecArgs::class(&vm.ctx).to_owned(), - "ParamSpecKwargs" => ParamSpecKwargs::class(&vm.ctx).to_owned(), - "Generic" => Generic::class(&vm.ctx).to_owned(), - "Union" => vm.ctx.types.union_type.to_owned(), - }); - module -} - #[pymodule(name = "_typing")] pub(crate) mod decl { use crate::{ Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, builtins::{PyStrRef, PyTupleRef, PyType, PyTypeRef, pystr::AsPyStr, type_}, + class::PyClassImpl, function::{FuncArgs, IntoFuncArgs}, protocol::PyNumberMethods, types::{AsNumber, Constructor, Representable}, @@ -206,4 +187,33 @@ pub(crate) mod decl { // &AS_MAPPING // } // } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py, + ) -> PyResult<()> { + __module_exec(vm, module); + + use super::{Generic, ParamSpec, ParamSpecArgs, ParamSpecKwargs, TypeVar, TypeVarTuple}; + + TypeVar::make_class(&vm.ctx); + ParamSpec::make_class(&vm.ctx); + TypeVarTuple::make_class(&vm.ctx); + ParamSpecArgs::make_class(&vm.ctx); + ParamSpecKwargs::make_class(&vm.ctx); + Generic::make_class(&vm.ctx); + + extend_module!(vm, module, { + "NoDefault" => vm.ctx.typing_no_default.clone(), + "TypeVar" => TypeVar::class(&vm.ctx).to_owned(), + "ParamSpec" => ParamSpec::class(&vm.ctx).to_owned(), + "TypeVarTuple" => TypeVarTuple::class(&vm.ctx).to_owned(), + "ParamSpecArgs" => ParamSpecArgs::class(&vm.ctx).to_owned(), + "ParamSpecKwargs" => ParamSpecKwargs::class(&vm.ctx).to_owned(), + "Generic" => Generic::class(&vm.ctx).to_owned(), + "Union" => vm.ctx.types.union_type.to_owned(), + }); + + Ok(()) + } } diff --git a/crates/vm/src/stdlib/warnings.rs b/crates/vm/src/stdlib/warnings.rs index 2d61c3b571f..ad54b806220 100644 --- a/crates/vm/src/stdlib/warnings.rs +++ b/crates/vm/src/stdlib/warnings.rs @@ -1,4 +1,4 @@ -pub(crate) use _warnings::make_module; +pub(crate) use _warnings::module_def; use crate::{Py, PyResult, VirtualMachine, builtins::PyType}; diff --git a/crates/vm/src/stdlib/weakref.rs b/crates/vm/src/stdlib/weakref.rs index bedfad9abbd..e7e030b2b01 100644 --- a/crates/vm/src/stdlib/weakref.rs +++ b/crates/vm/src/stdlib/weakref.rs @@ -4,7 +4,7 @@ //! - [python weakref module](https://docs.python.org/3/library/weakref.html) //! - [rust weak struct](https://doc.rust-lang.org/std/rc/struct.Weak.html) //! -pub(crate) use _weakref::make_module; +pub(crate) use _weakref::module_def; #[pymodule] mod _weakref { diff --git a/crates/vm/src/stdlib/winapi.rs b/crates/vm/src/stdlib/winapi.rs index 5145989e97f..a68ad3fbc58 100644 --- a/crates/vm/src/stdlib/winapi.rs +++ b/crates/vm/src/stdlib/winapi.rs @@ -1,7 +1,7 @@ // spell-checker:disable #![allow(non_snake_case)] -pub(crate) use _winapi::make_module; +pub(crate) use _winapi::module_def; #[pymodule] mod _winapi { diff --git a/crates/vm/src/stdlib/winreg.rs b/crates/vm/src/stdlib/winreg.rs index 28fa2e9b74c..53a4fd0d556 100644 --- a/crates/vm/src/stdlib/winreg.rs +++ b/crates/vm/src/stdlib/winreg.rs @@ -1,11 +1,7 @@ // spell-checker:disable #![allow(non_snake_case)] -use crate::{PyRef, VirtualMachine, builtins::PyModule}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - winreg::make_module(vm) -} +pub(crate) use winreg::module_def; #[pymodule] mod winreg { diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index 567bc40a2c3..7f33a8a49e4 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -1,7 +1,276 @@ -use super::{Context, PyConfig, VirtualMachine, setting::Settings, thread}; -use crate::{PyResult, getpath, stdlib::atexit, vm::PyBaseExceptionRef}; +use super::{Context, PyConfig, PyGlobalState, VirtualMachine, setting::Settings, thread}; +use crate::{ + PyResult, builtins, common::rc::PyRc, frozen::FrozenModule, getpath, py_freeze, stdlib::atexit, + vm::PyBaseExceptionRef, +}; use core::sync::atomic::Ordering; +type InitFunc = Box; + +/// Configuration builder for constructing an Interpreter. +/// +/// This is the preferred way to configure and create an interpreter with custom modules. +/// Modules must be registered before the interpreter is built, +/// similar to CPython's `PyImport_AppendInittab` which must be called before `Py_Initialize`. +/// +/// # Example +/// ``` +/// use rustpython_vm::Interpreter; +/// +/// let builder = Interpreter::builder(Default::default()); +/// // In practice, add stdlib: builder.add_native_modules(&stdlib_module_defs(&builder.ctx)) +/// let interp = builder.build(); +/// ``` +pub struct InterpreterBuilder { + settings: Settings, + pub ctx: PyRc, + module_defs: Vec<&'static builtins::PyModuleDef>, + frozen_modules: Vec<(&'static str, FrozenModule)>, + init_hooks: Vec, +} + +/// Private helper to initialize a VM with settings, context, and custom initialization. +fn initialize_main_vm( + settings: Settings, + ctx: PyRc, + module_defs: Vec<&'static builtins::PyModuleDef>, + frozen_modules: Vec<(&'static str, FrozenModule)>, + init_hooks: Vec, + init: F, +) -> (VirtualMachine, PyRc) +where + F: FnOnce(&mut VirtualMachine), +{ + use crate::codecs::CodecsRegistry; + use crate::common::hash::HashSecret; + use crate::common::lock::PyMutex; + use crate::warn::WarningsState; + use crossbeam_utils::atomic::AtomicCell; + use std::sync::atomic::AtomicBool; + + let paths = getpath::init_path_config(&settings); + let config = PyConfig::new(settings, paths); + + crate::types::TypeZoo::extend(&ctx); + crate::exceptions::ExceptionZoo::extend(&ctx); + + // Build module_defs map from builtin modules + additional modules + let mut all_module_defs: std::collections::BTreeMap< + &'static str, + &'static builtins::PyModuleDef, + > = crate::stdlib::builtin_module_defs(&ctx) + .into_iter() + .chain(module_defs) + .map(|def| (def.name.as_str(), def)) + .collect(); + + // Register sysconfigdata under platform-specific name as well + if let Some(&sysconfigdata_def) = all_module_defs.get("_sysconfigdata") { + use std::sync::OnceLock; + static SYSCONFIGDATA_NAME: OnceLock<&'static str> = OnceLock::new(); + let leaked_name = *SYSCONFIGDATA_NAME.get_or_init(|| { + let name = crate::stdlib::sys::sysconfigdata_name(); + Box::leak(name.into_boxed_str()) + }); + all_module_defs.insert(leaked_name, sysconfigdata_def); + } + + // Create hash secret + let seed = match config.settings.hash_seed { + Some(seed) => seed, + None => super::process_hash_secret_seed(), + }; + let hash_secret = HashSecret::new(seed); + + // Create codec registry and warnings state + let codec_registry = CodecsRegistry::new(&ctx); + let warnings = WarningsState::init_state(&ctx); + + // Create int_max_str_digits + let int_max_str_digits = AtomicCell::new(match config.settings.int_max_str_digits { + -1 => 4300, + other => other, + } as usize); + + // Initialize frozen modules (core + user-provided) + let mut frozen: std::collections::HashMap<&'static str, FrozenModule, ahash::RandomState> = + core_frozen_inits().collect(); + frozen.extend(frozen_modules); + + // Create PyGlobalState + let global_state = PyRc::new(PyGlobalState { + config, + module_defs: all_module_defs, + frozen, + stacksize: AtomicCell::new(0), + thread_count: AtomicCell::new(0), + hash_secret, + atexit_funcs: PyMutex::default(), + codec_registry, + finalizing: AtomicBool::new(false), + warnings, + override_frozen_modules: AtomicCell::new(0), + before_forkers: PyMutex::default(), + after_forkers_child: PyMutex::default(), + after_forkers_parent: PyMutex::default(), + int_max_str_digits, + switch_interval: AtomicCell::new(0.005), + global_trace_func: PyMutex::default(), + global_profile_func: PyMutex::default(), + #[cfg(feature = "threading")] + main_thread_ident: AtomicCell::new(0), + #[cfg(feature = "threading")] + thread_frames: parking_lot::Mutex::new(std::collections::HashMap::new()), + #[cfg(feature = "threading")] + thread_handles: parking_lot::Mutex::new(Vec::new()), + #[cfg(feature = "threading")] + shutdown_handles: parking_lot::Mutex::new(Vec::new()), + }); + + // Create VM with the global state + // Note: Don't clone here - init_hooks need exclusive access to mutate state + let mut vm = VirtualMachine::new(ctx, global_state); + + // Execute initialization hooks (can mutate vm.state) + for hook in init_hooks { + hook(&mut vm); + } + + // Call custom init function (can mutate vm.state) + init(&mut vm); + + vm.initialize(); + + // Clone global_state for Interpreter after all initialization is done + let global_state = vm.state.clone(); + (vm, global_state) +} + +impl InterpreterBuilder { + /// Create a new interpreter configuration with default settings. + pub fn new() -> Self { + Self { + settings: Settings::default(), + ctx: Context::genesis().clone(), + module_defs: Vec::new(), + frozen_modules: Vec::new(), + init_hooks: Vec::new(), + } + } + + /// Set custom settings for the interpreter. + /// + /// If called multiple times, only the last settings will be used. + pub fn settings(mut self, settings: Settings) -> Self { + self.settings = settings; + self + } + + /// Add a single native module definition. + /// + /// # Example + /// ``` + /// use rustpython_vm::{Interpreter, builtins::PyModuleDef}; + /// + /// let builder = Interpreter::builder(Default::default()); + /// // Note: In practice, use module_def from your #[pymodule] + /// // let def = mymodule::module_def(&builder.ctx); + /// // let interp = builder.add_native_module(def).build(); + /// let interp = builder.build(); + /// ``` + pub fn add_native_module(self, def: &'static builtins::PyModuleDef) -> Self { + self.add_native_modules(&[def]) + } + + /// Add multiple native module definitions. + /// + /// # Example + /// ``` + /// use rustpython_vm::Interpreter; + /// + /// let builder = Interpreter::builder(Default::default()); + /// // In practice, use module_defs from rustpython_stdlib: + /// // let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + /// // let interp = builder.add_native_modules(&defs).build(); + /// let interp = builder.build(); + /// ``` + pub fn add_native_modules(mut self, defs: &[&'static builtins::PyModuleDef]) -> Self { + self.module_defs.extend_from_slice(defs); + self + } + + /// Add a custom initialization hook. + /// + /// Hooks are executed in the order they are added during interpreter creation. + /// This function will be called after modules are registered but before + /// the VM is initialized, allowing for additional customization. + /// + /// # Example + /// ``` + /// use rustpython_vm::Interpreter; + /// + /// let interp = Interpreter::builder(Default::default()) + /// .init_hook(|vm| { + /// // Custom initialization + /// }) + /// .build(); + /// ``` + pub fn init_hook(mut self, init: F) -> Self + where + F: FnOnce(&mut VirtualMachine) + 'static, + { + self.init_hooks.push(Box::new(init)); + self + } + + /// Add frozen modules to the interpreter. + /// + /// Frozen modules are Python modules compiled into the binary. + /// This method accepts any iterator of (name, FrozenModule) pairs. + /// + /// # Example + /// ``` + /// use rustpython_vm::Interpreter; + /// + /// let interp = Interpreter::builder(Default::default()) + /// // In practice: .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB) + /// .build(); + /// ``` + pub fn add_frozen_modules(mut self, frozen: I) -> Self + where + I: IntoIterator, + { + self.frozen_modules.extend(frozen); + self + } + + /// Build the interpreter. + /// + /// This consumes the configuration and returns a fully initialized Interpreter. + pub fn build(self) -> Interpreter { + let (vm, global_state) = initialize_main_vm( + self.settings, + self.ctx, + self.module_defs, + self.frozen_modules, + self.init_hooks, + |_| {}, // No additional init needed + ); + Interpreter { global_state, vm } + } + + /// Alias for `build()` for compatibility with the `interpreter()` pattern. + pub fn interpreter(self) -> Interpreter { + self.build() + } +} + +impl Default for InterpreterBuilder { + fn default() -> Self { + Self::new() + } +} + /// The general interface for the VM /// /// # Examples @@ -21,43 +290,49 @@ use core::sync::atomic::Ordering; /// }); /// ``` pub struct Interpreter { + pub global_state: PyRc, vm: VirtualMachine, } impl Interpreter { + /// Create a new interpreter configuration builder. + /// + /// # Example + /// ``` + /// use rustpython_vm::Interpreter; + /// + /// let builder = Interpreter::builder(Default::default()); + /// // In practice, add stdlib: builder.add_native_modules(&stdlib_module_defs(&builder.ctx)) + /// let interp = builder.build(); + /// ``` + pub fn builder(settings: Settings) -> InterpreterBuilder { + InterpreterBuilder::new().settings(settings) + } + /// This is a bare unit to build up an interpreter without the standard library. - /// To create an interpreter with the standard library with the `rustpython` crate, use `rustpython::InterpreterConfig`. + /// To create an interpreter with the standard library with the `rustpython` crate, use `rustpython::InterpreterBuilder`. /// To create an interpreter without the `rustpython` crate, but only with `rustpython-vm`, - /// try to build one from the source code of `InterpreterConfig`. It will not be a one-liner but it also will not be too hard. + /// try to build one from the source code of `InterpreterBuilder`. It will not be a one-liner but it also will not be too hard. pub fn without_stdlib(settings: Settings) -> Self { Self::with_init(settings, |_| {}) } /// Create with initialize function taking mutable vm reference. - /// ``` - /// use rustpython_vm::Interpreter; - /// Interpreter::with_init(Default::default(), |vm| { - /// // put this line to add stdlib to the vm - /// // vm.add_native_modules(rustpython_stdlib::get_module_inits()); - /// }).enter(|vm| { - /// vm.run_code_string(vm.new_scope_with_builtins(), "print(1)", "<...>".to_owned()); - /// }); - /// ``` + /// + /// Note: This is a legacy API. To add stdlib, use `Interpreter::builder()` instead. pub fn with_init(settings: Settings, init: F) -> Self where F: FnOnce(&mut VirtualMachine), { - // Compute path configuration from settings - let paths = getpath::init_path_config(&settings); - let config = PyConfig::new(settings, paths); - - let ctx = Context::genesis(); - crate::types::TypeZoo::extend(ctx); - crate::exceptions::ExceptionZoo::extend(ctx); - let mut vm = VirtualMachine::new(config, ctx.clone()); - init(&mut vm); - vm.initialize(); - Self { vm } + let (vm, global_state) = initialize_main_vm( + settings, + Context::genesis().clone(), + Vec::new(), // No module_defs + Vec::new(), // No frozen_modules + Vec::new(), // No init_hooks + init, + ); + Self { global_state, vm } } /// Run a function with the main virtual machine and return a PyResult of the result. @@ -157,6 +432,44 @@ impl Interpreter { } } +fn core_frozen_inits() -> impl Iterator { + let iter = core::iter::empty(); + macro_rules! ext_modules { + ($iter:ident, $($t:tt)*) => { + let $iter = $iter.chain(py_freeze!($($t)*)); + }; + } + + // keep as example but use file one now + // ext_modules!( + // iter, + // source = "initialized = True; print(\"Hello world!\")\n", + // module_name = "__hello__", + // ); + + // Python modules that the vm calls into, but are not actually part of the stdlib. They could + // in theory be implemented in Rust, but are easiest to do in Python for one reason or another. + // Includes _importlib_bootstrap and _importlib_bootstrap_external + ext_modules!( + iter, + dir = "./Lib/python_builtins", + crate_name = "rustpython_compiler_core" + ); + + // core stdlib Python modules that the vm calls into, but are still used in Python + // application code, e.g. copyreg + // FIXME: Initializing core_modules here results duplicated frozen module generation for core_modules. + // We need a way to initialize this modules for both `Interpreter::without_stdlib()` and `InterpreterBuilder::new().init_stdlib().interpreter()` + // #[cfg(not(feature = "freeze-stdlib"))] + ext_modules!( + iter, + dir = "./Lib/core_modules", + crate_name = "rustpython_compiler_core" + ); + + iter +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 5bd2b9e8297..183c7d294db 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -19,7 +19,7 @@ mod vm_ops; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, builtins::{ - PyBaseExceptionRef, PyDict, PyDictRef, PyInt, PyList, PyModule, PyStr, PyStrInterned, + self, PyBaseExceptionRef, PyDict, PyDictRef, PyInt, PyList, PyModule, PyStr, PyStrInterned, PyStrRef, PyTypeRef, code::PyCode, dict::{PyDictItems, PyDictKeys, PyDictValues}, @@ -56,7 +56,7 @@ use std::{ }; pub use context::Context; -pub use interpreter::Interpreter; +pub use interpreter::{Interpreter, InterpreterBuilder}; pub(crate) use method::PyMethod; pub use setting::{CheckHashPycsMode, Paths, PyConfig, Settings}; @@ -102,7 +102,7 @@ struct ExceptionStack { pub struct PyGlobalState { pub config: PyConfig, - pub module_inits: stdlib::StdlibMap, + pub module_defs: std::collections::BTreeMap<&'static str, &'static builtins::PyModuleDef>, pub frozen: HashMap<&'static str, FrozenModule, ahash::RandomState>, pub stacksize: AtomicCell, pub thread_count: AtomicCell, @@ -144,7 +144,7 @@ pub fn process_hash_secret_seed() -> u32 { impl VirtualMachine { /// Create a new `VirtualMachine` structure. - fn new(config: PyConfig, ctx: PyRc) -> Self { + pub(crate) fn new(ctx: PyRc, state: PyRc) -> Self { flame_guard!("new VirtualMachine"); // make a new module without access to the vm; doesn't @@ -158,8 +158,8 @@ impl VirtualMachine { }; // Hard-core modules: - let builtins = new_module(stdlib::builtins::__module_def(&ctx)); - let sys_module = new_module(stdlib::sys::__module_def(&ctx)); + let builtins = new_module(stdlib::builtins::module_def(&ctx)); + let sys_module = new_module(stdlib::sys::module_def(&ctx)); let import_func = ctx.none(); let profile_func = RefCell::new(ctx.none()); @@ -169,23 +169,7 @@ impl VirtualMachine { const { RefCell::new([const { None }; signal::NSIG]) }, )); - let module_inits = stdlib::get_module_inits(); - - let seed = match config.settings.hash_seed { - Some(seed) => seed, - None => process_hash_secret_seed(), - }; - let hash_secret = HashSecret::new(seed); - - let codec_registry = CodecsRegistry::new(&ctx); - - let warnings = WarningsState::init_state(&ctx); - - let int_max_str_digits = AtomicCell::new(match config.settings.int_max_str_digits { - -1 => 4300, - other => other, - } as usize); - let mut vm = Self { + let vm = Self { builtins, sys_module, ctx, @@ -200,34 +184,7 @@ impl VirtualMachine { signal_handlers, signal_rx: None, repr_guards: RefCell::default(), - state: PyRc::new(PyGlobalState { - config, - module_inits, - frozen: HashMap::default(), - stacksize: AtomicCell::new(0), - thread_count: AtomicCell::new(0), - hash_secret, - atexit_funcs: PyMutex::default(), - codec_registry, - finalizing: AtomicBool::new(false), - warnings, - override_frozen_modules: AtomicCell::new(0), - before_forkers: PyMutex::default(), - after_forkers_child: PyMutex::default(), - after_forkers_parent: PyMutex::default(), - int_max_str_digits, - switch_interval: AtomicCell::new(0.005), - global_trace_func: PyMutex::default(), - global_profile_func: PyMutex::default(), - #[cfg(feature = "threading")] - main_thread_ident: AtomicCell::new(0), - #[cfg(feature = "threading")] - thread_frames: parking_lot::Mutex::new(HashMap::new()), - #[cfg(feature = "threading")] - thread_handles: parking_lot::Mutex::new(Vec::new()), - #[cfg(feature = "threading")] - shutdown_handles: parking_lot::Mutex::new(Vec::new()), - }), + state, initialized: false, recursion_depth: Cell::new(0), c_stack_soft_limit: Cell::new(Self::calculate_c_stack_soft_limit()), @@ -245,9 +202,6 @@ impl VirtualMachine { panic!("Interpreters in same process must share the hash seed"); } - let frozen = core_frozen_inits().collect(); - PyRc::get_mut(&mut vm.state).unwrap().frozen = frozen; - vm.builtins.init_dict( vm.ctx.intern_str("builtins"), Some(vm.ctx.intern_str(stdlib::builtins::DOC.unwrap()).to_owned()), @@ -275,7 +229,7 @@ impl VirtualMachine { }); let guide_message = if cfg!(feature = "freeze-stdlib") { - "`rustpython_pylib` maybe not set while using `freeze-stdlib` feature. Try using `rustpython::InterpreterConfig::init_stdlib` or manually call `vm.add_frozen(rustpython_pylib::FROZEN_STDLIB)` in `rustpython_vm::Interpreter::with_init`." + "`rustpython_pylib` may not be set while using `freeze-stdlib` feature. Try using `rustpython::InterpreterBuilder::init_stdlib` or manually call `builder.add_frozen_modules(rustpython_pylib::FROZEN_STDLIB)` in `rustpython_vm::Interpreter::builder()`." } else if !env_set { "Neither RUSTPYTHONPATH nor PYTHONPATH is set. Try setting one of them to the stdlib directory." } else if path_contains_env { @@ -470,34 +424,6 @@ impl VirtualMachine { self.initialized = true; } - fn state_mut(&mut self) -> &mut PyGlobalState { - PyRc::get_mut(&mut self.state) - .expect("there should not be multiple threads while a user has a mut ref to a vm") - } - - /// Can only be used in the initialization closure passed to [`Interpreter::with_init`] - pub fn add_native_module(&mut self, name: S, module: stdlib::StdlibInitFunc) - where - S: Into>, - { - self.state_mut().module_inits.insert(name.into(), module); - } - - pub fn add_native_modules(&mut self, iter: I) - where - I: IntoIterator, stdlib::StdlibInitFunc)>, - { - self.state_mut().module_inits.extend(iter); - } - - /// Can only be used in the initialization closure passed to [`Interpreter::with_init`] - pub fn add_frozen(&mut self, frozen: I) - where - I: IntoIterator, - { - self.state_mut().frozen.extend(frozen); - } - /// Set the custom signal channel for the interpreter pub fn set_user_signal_channel(&mut self, signal_rx: signal::UserSignalReceiver) { self.signal_rx = Some(signal_rx); @@ -1381,89 +1307,52 @@ pub fn resolve_frozen_alias(name: &str) -> &str { } } -fn core_frozen_inits() -> impl Iterator { - let iter = core::iter::empty(); - macro_rules! ext_modules { - ($iter:ident, $($t:tt)*) => { - let $iter = $iter.chain(py_freeze!($($t)*)); - }; - } - - // keep as example but use file one now - // ext_modules!( - // iter, - // source = "initialized = True; print(\"Hello world!\")\n", - // module_name = "__hello__", - // ); - - // Python modules that the vm calls into, but are not actually part of the stdlib. They could - // in theory be implemented in Rust, but are easiest to do in Python for one reason or another. - // Includes _importlib_bootstrap and _importlib_bootstrap_external - ext_modules!( - iter, - dir = "./Lib/python_builtins", - crate_name = "rustpython_compiler_core" - ); - - // core stdlib Python modules that the vm calls into, but are still used in Python - // application code, e.g. copyreg - // FIXME: Initializing core_modules here results duplicated frozen module generation for core_modules. - // We need a way to initialize this modules for both `Interpreter::without_stdlib()` and `InterpreterConfig::new().init_stdlib().interpreter()` - // #[cfg(not(feature = "freeze-stdlib"))] - ext_modules!( - iter, - dir = "./Lib/core_modules", - crate_name = "rustpython_compiler_core" - ); - - iter -} - #[test] fn test_nested_frozen() { use rustpython_vm as vm; - vm::Interpreter::with_init(Default::default(), |vm| { - // vm.add_native_modules(rustpython_stdlib::get_module_inits()); - vm.add_frozen(rustpython_vm::py_freeze!( + vm::Interpreter::builder(Default::default()) + .add_frozen_modules(rustpython_vm::py_freeze!( dir = "../../extra_tests/snippets" - )); - }) - .enter(|vm| { - let scope = vm.new_scope_with_builtins(); - - let source = "from dir_module.dir_module_inner import value2"; - let code_obj = vm - .compile(source, vm::compiler::Mode::Exec, "".to_owned()) - .map_err(|err| vm.new_syntax_error(&err, Some(source))) - .unwrap(); - - if let Err(e) = vm.run_code_obj(code_obj, scope) { - vm.print_exception(e); - panic!(); - } - }) + )) + .build() + .enter(|vm| { + let scope = vm.new_scope_with_builtins(); + + let source = "from dir_module.dir_module_inner import value2"; + let code_obj = vm + .compile(source, vm::compiler::Mode::Exec, "".to_owned()) + .map_err(|err| vm.new_syntax_error(&err, Some(source))) + .unwrap(); + + if let Err(e) = vm.run_code_obj(code_obj, scope) { + vm.print_exception(e); + panic!(); + } + }) } #[test] fn frozen_origname_matches() { use rustpython_vm as vm; - vm::Interpreter::with_init(Default::default(), |_vm| {}).enter(|vm| { - let check = |name, expected| { - let module = import::import_frozen(vm, name).unwrap(); - let origname: PyStrRef = module - .get_attr("__origname__", vm) - .unwrap() - .try_into_value(vm) - .unwrap(); - assert_eq!(origname.as_str(), expected); - }; + vm::Interpreter::builder(Default::default()) + .build() + .enter(|vm| { + let check = |name, expected| { + let module = import::import_frozen(vm, name).unwrap(); + let origname: PyStrRef = module + .get_attr("__origname__", vm) + .unwrap() + .try_into_value(vm) + .unwrap(); + assert_eq!(origname.as_str(), expected); + }; - check("_frozen_importlib", "importlib._bootstrap"); - check( - "_frozen_importlib_external", - "importlib._bootstrap_external", - ); - }); + check("_frozen_importlib", "importlib._bootstrap"); + check( + "_frozen_importlib_external", + "importlib._bootstrap_external", + ); + }); } diff --git a/crates/wasm/src/browser_module.rs b/crates/wasm/src/browser_module.rs index 9b6219cab6f..d1eecce28a9 100644 --- a/crates/wasm/src/browser_module.rs +++ b/crates/wasm/src/browser_module.rs @@ -1,6 +1,4 @@ -use rustpython_vm::VirtualMachine; - -pub(crate) use _browser::make_module; +pub(crate) use _browser::module_def; #[pymodule] mod _browser { @@ -257,8 +255,3 @@ mod _browser { Ok(PyPromise::from_future(future).into_pyobject(vm)) } } - -pub fn setup_browser_module(vm: &mut VirtualMachine) { - vm.add_native_module("_browser".to_owned(), Box::new(make_module)); - vm.add_frozen(py_freeze!(dir = "Lib")); -} diff --git a/crates/wasm/src/js_module.rs b/crates/wasm/src/js_module.rs index 11d23bdf6d8..750e85994a1 100644 --- a/crates/wasm/src/js_module.rs +++ b/crates/wasm/src/js_module.rs @@ -1,5 +1,4 @@ pub(crate) use _js::{PyJsValue, PyPromise}; -use rustpython_vm::VirtualMachine; #[pymodule] mod _js { @@ -621,8 +620,4 @@ mod _js { } } -pub(crate) use _js::make_module; - -pub fn setup_js_module(vm: &mut VirtualMachine) { - vm.add_native_module("_js".to_owned(), Box::new(make_module)); -} +pub(crate) use _js::module_def; diff --git a/crates/wasm/src/lib.rs b/crates/wasm/src/lib.rs index 64595fd63ab..99668df2855 100644 --- a/crates/wasm/src/lib.rs +++ b/crates/wasm/src/lib.rs @@ -11,7 +11,6 @@ extern crate rustpython_vm; use js_sys::{Reflect, WebAssembly::RuntimeError}; use std::panic; -pub use vm_class::add_init_func; pub(crate) use vm_class::weak_vm; use wasm_bindgen::prelude::*; diff --git a/crates/wasm/src/vm_class.rs b/crates/wasm/src/vm_class.rs index 0339a47821e..6baef990d1e 100644 --- a/crates/wasm/src/vm_class.rs +++ b/crates/wasm/src/vm_class.rs @@ -1,5 +1,5 @@ use crate::{ - browser_module::setup_browser_module, + browser_module, convert::{self, PyResultExt}, js_module, wasm_builtins, }; @@ -7,10 +7,8 @@ use alloc::rc::{Rc, Weak}; use core::cell::RefCell; use js_sys::{Object, TypeError}; use rustpython_vm::{ - Interpreter, PyObjectRef, PyPayload, PyRef, PyResult, Settings, VirtualMachine, - builtins::{PyModule, PyWeak}, - compiler::Mode, - scope::Scope, + Interpreter, PyObjectRef, PyRef, PyResult, Settings, VirtualMachine, builtins::PyWeak, + compiler::Mode, scope::Scope, }; use std::collections::HashMap; use wasm_bindgen::prelude::*; @@ -24,69 +22,67 @@ pub(crate) struct StoredVirtualMachine { } #[pymodule] -mod _window {} - -fn init_window_module(vm: &VirtualMachine) -> PyRef { - let module = _window::make_module(vm); - - extend_module!(vm, &module, { - "window" => js_module::PyJsValue::new(wasm_builtins::window()).into_ref(&vm.ctx), - }); - - module +mod _window { + use super::{js_module, wasm_builtins}; + use rustpython_vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + __module_exec(vm, module); + extend_module!(vm, module, { + "window" => js_module::PyJsValue::new(wasm_builtins::window()).into_ref(&vm.ctx), + }); + Ok(()) + } } impl StoredVirtualMachine { fn new(id: String, inject_browser_module: bool) -> StoredVirtualMachine { - let mut scope = None; let mut settings = Settings::default(); settings.allow_external_library = false; - let interp = Interpreter::with_init(settings, |vm| { - #[cfg(feature = "freeze-stdlib")] - vm.add_native_modules(rustpython_stdlib::get_module_inits()); - #[cfg(feature = "freeze-stdlib")] - vm.add_frozen(rustpython_pylib::FROZEN_STDLIB); + let mut builder = Interpreter::builder(settings); + + #[cfg(feature = "freeze-stdlib")] + { + let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + builder = builder + .add_native_modules(&defs) + .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB); + } - vm.wasm_id = Some(id); + // Add wasm-specific modules + let js_def = js_module::module_def(&builder.ctx); + builder = builder.add_native_module(js_def); - js_module::setup_js_module(vm); - if inject_browser_module { - vm.add_native_module("_window".to_owned(), Box::new(init_window_module)); - setup_browser_module(vm); - } + if inject_browser_module { + let window_def = _window::module_def(&builder.ctx); + let browser_def = browser_module::module_def(&builder.ctx); + builder = builder + .add_native_modules(&[window_def, browser_def]) + .add_frozen_modules(rustpython_vm::py_freeze!(dir = "Lib")); + } - VM_INIT_FUNCS.with_borrow(|funcs| { - for f in funcs { - f(vm) - } - }); + let interp = builder + .init_hook(move |vm| { + vm.wasm_id = Some(id); + }) + .build(); - scope = Some(vm.new_scope_with_builtins()); - }); + let scope = interp.enter(|vm| vm.new_scope_with_builtins()); StoredVirtualMachine { interp, - scope: scope.unwrap(), + scope, held_objects: RefCell::new(Vec::new()), } } } -/// Add a hook to add builtins or frozen modules to the RustPython VirtualMachine while it's -/// initializing. -pub fn add_init_func(f: fn(&mut VirtualMachine)) { - VM_INIT_FUNCS.with_borrow_mut(|funcs| funcs.push(f)) -} - // It's fine that it's thread local, since WASM doesn't even have threads yet. thread_local! // probably gets compiled down to a normal-ish static variable, like Atomic* types do: // https://rustwasm.github.io/2018/10/24/multithreading-rust-and-wasm.html#atomic-instructions thread_local! { static STORED_VMS: RefCell>> = RefCell::default(); - static VM_INIT_FUNCS: RefCell> = const { - RefCell::new(Vec::new()) - }; } pub fn get_vm_id(vm: &VirtualMachine) -> &str { diff --git a/example_projects/frozen_stdlib/src/main.rs b/example_projects/frozen_stdlib/src/main.rs index 2688d2164a2..8ff316faed8 100644 --- a/example_projects/frozen_stdlib/src/main.rs +++ b/example_projects/frozen_stdlib/src/main.rs @@ -1,9 +1,10 @@ // spell-checker:ignore aheui -/// Setting up a project with a frozen stdlib can be done *either* by using `rustpython::InterpreterConfig` or `rustpython_vm::Interpreter::with_init`. -/// See each function for example. -/// -/// See also: `aheui-rust.md` for freezing your own package. +//! Setting up a project with a frozen stdlib can be done *either* by using `rustpython::InterpreterBuilder` or `rustpython_vm::Interpreter::builder`. +//! See each function for example. +//! +//! See also: `aheui-rust.md` for freezing your own package. +use rustpython::InterpreterBuilderExt; use rustpython_vm::{PyResult, VirtualMachine}; fn run(keyword: &str, vm: &VirtualMachine) -> PyResult<()> { @@ -17,21 +18,19 @@ fn run(keyword: &str, vm: &VirtualMachine) -> PyResult<()> { } fn interpreter_with_config() { - let interpreter = rustpython::InterpreterConfig::new() + let interpreter = rustpython::InterpreterBuilder::new() .init_stdlib() .interpreter(); // Use interpreter.enter to reuse the same interpreter later - interpreter.run(|vm| run("rustpython::InterpreterConfig", vm)); + interpreter.run(|vm| run("rustpython::InterpreterBuilder", vm)); } fn interpreter_with_vm() { - let interpreter = rustpython_vm::Interpreter::with_init(Default::default(), |vm| { - // This is unintuitive, but the stdlib is out of the vm crate. - // Any suggestion to improve this is welcome. - vm.add_frozen(rustpython_pylib::FROZEN_STDLIB); - }); + let interpreter = rustpython_vm::Interpreter::builder(Default::default()) + .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB) + .build(); // Use interpreter.enter to reuse the same interpreter later - interpreter.run(|vm| run("rustpython_vm::Interpreter::with_init", vm)); + interpreter.run(|vm| run("rustpython_vm::Interpreter::builder", vm)); } fn main() { diff --git a/examples/call_between_rust_and_python.rs b/examples/call_between_rust_and_python.rs index dee17058475..3a7c0ce610e 100644 --- a/examples/call_between_rust_and_python.rs +++ b/examples/call_between_rust_and_python.rs @@ -1,17 +1,12 @@ +use rustpython::InterpreterBuilderExt; use rustpython::vm::{ PyObject, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, pyclass, pymodule, }; pub fn main() { - let interp = rustpython::InterpreterConfig::new() - .init_stdlib() - .init_hook(Box::new(|vm| { - vm.add_native_module( - "rust_py_module".to_owned(), - Box::new(rust_py_module::make_module), - ); - })) - .interpreter(); + let builder = rustpython::Interpreter::builder(Default::default()); + let def = rust_py_module::module_def(&builder.ctx); + let interp = builder.init_stdlib().add_native_module(def).build(); interp.enter(|vm| { vm.insert_sys_path(vm.new_pyobj("examples")) diff --git a/examples/generator.rs b/examples/generator.rs index 27733a1913d..55841c767a1 100644 --- a/examples/generator.rs +++ b/examples/generator.rs @@ -42,9 +42,9 @@ gen() } fn main() -> ExitCode { - let interp = vm::Interpreter::with_init(Default::default(), |vm| { - vm.add_native_modules(rustpython_stdlib::get_module_inits()); - }); + let builder = vm::Interpreter::builder(Default::default()); + let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + let interp = builder.add_native_modules(&defs).build(); let result = py_main(&interp); vm::common::os::exit_code(interp.run(|_vm| result)) } diff --git a/examples/package_embed.rs b/examples/package_embed.rs index e82e71f5ceb..bb2f29e3f5f 100644 --- a/examples/package_embed.rs +++ b/examples/package_embed.rs @@ -19,9 +19,9 @@ fn main() -> ExitCode { // Add standard library path let mut settings = vm::Settings::default(); settings.path_list.push("Lib".to_owned()); - let interp = vm::Interpreter::with_init(settings, |vm| { - vm.add_native_modules(rustpython_stdlib::get_module_inits()); - }); + let builder = vm::Interpreter::builder(settings); + let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + let interp = builder.add_native_modules(&defs).build(); let result = py_main(&interp); let result = result.map(|result| { println!("name: {result}"); diff --git a/src/interpreter.rs b/src/interpreter.rs index 08d23e78afe..b9ee2dbbc44 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -1,130 +1,44 @@ -use rustpython_vm::{Interpreter, PyRef, Settings, VirtualMachine, builtins::PyModule}; +use rustpython_vm::InterpreterBuilder; -pub type InitHook = Box; - -/// The convenient way to create [rustpython_vm::Interpreter] with stdlib and other stuffs. -/// -/// Basic usage: -/// ``` -/// let interpreter = rustpython::InterpreterConfig::new() -/// .init_stdlib() -/// .interpreter(); -/// ``` -/// -/// To override [rustpython_vm::Settings]: -/// ``` -/// use rustpython_vm::Settings; -/// // Override your settings here. -/// let mut settings = Settings::default(); -/// settings.debug = 1; -/// // You may want to add paths to `rustpython_vm::Settings::path_list` to allow import python libraries. -/// settings.path_list.push("Lib".to_owned()); // add standard library directory -/// settings.path_list.push("".to_owned()); // add current working directory -/// let interpreter = rustpython::InterpreterConfig::new() -/// .settings(settings) -/// .interpreter(); -/// ``` -/// -/// To add native modules: -/// ``` -/// use rustpython_vm::pymodule; -/// -/// #[pymodule] -/// mod your_module {} -/// -/// let interpreter = rustpython::InterpreterConfig::new() -/// .init_stdlib() -/// .add_native_module( -/// "your_module_name".to_owned(), -/// your_module::make_module, -/// ) -/// .interpreter(); -/// ``` -#[derive(Default)] -pub struct InterpreterConfig { - settings: Option, - init_hooks: Vec, -} - -impl InterpreterConfig { - /// Creates a new interpreter configuration with default settings. - pub fn new() -> Self { - Self::default() - } - - /// Builds the interpreter with the current configuration. - pub fn interpreter(self) -> Interpreter { - let settings = self.settings.unwrap_or_default(); - Interpreter::with_init(settings, |vm| { - for hook in self.init_hooks { - hook(vm); - } - }) - } - - /// Sets custom settings for the interpreter. - /// - /// If called multiple times, only the last settings will be used. - pub fn settings(mut self, settings: Settings) -> Self { - self.settings = Some(settings); - self - } - - /// Adds a custom initialization hook. - /// - /// Hooks are executed in the order they are added during interpreter creation. - pub fn init_hook(mut self, hook: InitHook) -> Self { - self.init_hooks.push(hook); - self - } - - /// Adds a native module to the interpreter. - pub fn add_native_module( - self, - name: String, - make_module: fn(&VirtualMachine) -> PyRef, - ) -> Self { - self.init_hook(Box::new(move |vm| { - vm.add_native_module(name, Box::new(make_module)) - })) - } - - /// Initializes the Python standard library. +/// Extension trait for InterpreterBuilder to add rustpython-specific functionality. +pub trait InterpreterBuilderExt { + /// Initialize the Python standard library. /// /// Requires the `stdlib` feature to be enabled. #[cfg(feature = "stdlib")] - pub fn init_stdlib(self) -> Self { - self.init_hook(Box::new(init_stdlib)) - } + fn init_stdlib(self) -> Self; } -/// Initializes all standard library modules for the given VM. -#[cfg(feature = "stdlib")] -pub fn init_stdlib(vm: &mut VirtualMachine) { - vm.add_native_modules(rustpython_stdlib::get_module_inits()); +impl InterpreterBuilderExt for InterpreterBuilder { + #[cfg(feature = "stdlib")] + fn init_stdlib(self) -> Self { + let defs = rustpython_stdlib::stdlib_module_defs(&self.ctx); + let builder = self.add_native_modules(&defs); + + #[cfg(feature = "freeze-stdlib")] + let builder = builder + .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB) + .init_hook(set_frozen_stdlib_dir); - #[cfg(feature = "freeze-stdlib")] - setup_frozen_stdlib(vm); + #[cfg(not(feature = "freeze-stdlib"))] + let builder = builder.init_hook(setup_dynamic_stdlib); - #[cfg(not(feature = "freeze-stdlib"))] - setup_dynamic_stdlib(vm); + builder + } } -/// Setup frozen standard library (compiled into the binary) +/// Set stdlib_dir for frozen standard library #[cfg(all(feature = "stdlib", feature = "freeze-stdlib"))] -fn setup_frozen_stdlib(vm: &mut VirtualMachine) { +fn set_frozen_stdlib_dir(vm: &mut crate::VirtualMachine) { use rustpython_vm::common::rc::PyRc; - vm.add_frozen(rustpython_pylib::FROZEN_STDLIB); - - // Set stdlib_dir to the frozen stdlib path let state = PyRc::get_mut(&mut vm.state).unwrap(); state.config.paths.stdlib_dir = Some(rustpython_pylib::LIB_PATH.to_owned()); } /// Setup dynamic standard library loading from filesystem #[cfg(all(feature = "stdlib", not(feature = "freeze-stdlib")))] -fn setup_dynamic_stdlib(vm: &mut VirtualMachine) { +fn setup_dynamic_stdlib(vm: &mut crate::VirtualMachine) { use rustpython_vm::common::rc::PyRc; let state = PyRc::get_mut(&mut vm.state).unwrap(); diff --git a/src/lib.rs b/src/lib.rs index cf5227a70dd..8372a6d2e67 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,23 +1,29 @@ //! This is the `rustpython` binary. If you're looking to embed RustPython into your application, //! you're likely looking for the [`rustpython_vm`] crate. //! -//! You can install `rustpython` with `cargo install rustpython`, or if you'd like to inject your -//! own native modules you can make a binary crate that depends on the `rustpython` crate (and +//! You can install `rustpython` with `cargo install rustpython`. If you'd like to inject your +//! own native modules, you can make a binary crate that depends on the `rustpython` crate (and //! probably [`rustpython_vm`], too), and make a `main.rs` that looks like: //! //! ```no_run +//! use rustpython::{InterpreterBuilder, InterpreterBuilderExt}; //! use rustpython_vm::{pymodule, py_freeze}; -//! fn main() { -//! rustpython::run(|vm| { -//! vm.add_native_module("my_mod".to_owned(), Box::new(my_mod::make_module)); -//! vm.add_frozen(py_freeze!(source = "def foo(): pass", module_name = "other_thing")); -//! }); +//! +//! fn main() -> std::process::ExitCode { +//! let builder = InterpreterBuilder::new().init_stdlib(); +//! // Add a native module using builder.ctx +//! let my_mod_def = my_mod::module_def(&builder.ctx); +//! let builder = builder +//! .add_native_module(my_mod_def) +//! // Add a frozen module +//! .add_frozen_modules(py_freeze!(source = "def foo(): pass", module_name = "other_thing")); +//! +//! rustpython::run(builder) //! } //! //! #[pymodule] //! mod my_mod { //! use rustpython_vm::builtins::PyStrRef; -//TODO: use rustpython_vm::prelude::*; //! //! #[pyfunction] //! fn do_thing(x: i32) -> i32 { @@ -54,8 +60,8 @@ use std::env; use std::io::IsTerminal; use std::process::ExitCode; -pub use interpreter::InterpreterConfig; -pub use rustpython_vm as vm; +pub use interpreter::InterpreterBuilderExt; +pub use rustpython_vm::{self as vm, Interpreter, InterpreterBuilder}; pub use settings::{InstallPipMode, RunMode, parse_opts}; pub use shell::run_shell; @@ -69,7 +75,11 @@ compile_error!( /// The main cli of the `rustpython` interpreter. This function will return `std::process::ExitCode` /// based on the return code of the python code ran through the cli. -pub fn run(init: impl FnOnce(&mut VirtualMachine) + 'static) -> ExitCode { +/// +/// **Note**: This function provides no way to further initialize the VM after the builder is applied. +/// All VM initialization (adding native modules, init hooks, etc.) must be done through the +/// [`InterpreterBuilder`] parameter before calling this function. +pub fn run(mut builder: InterpreterBuilder) -> ExitCode { env_logger::init(); // NOTE: This is not a WASI convention. But it will be convenient since POSIX shell always defines it. @@ -101,14 +111,9 @@ pub fn run(init: impl FnOnce(&mut VirtualMachine) + 'static) -> ExitCode { } } - let mut config = InterpreterConfig::new().settings(settings); - #[cfg(feature = "stdlib")] - { - config = config.init_stdlib(); - } - config = config.init_hook(Box::new(init)); + builder = builder.settings(settings); - let interp = config.interpreter(); + let interp = builder.interpreter(); let exitcode = interp.run(move |vm| run_rustpython(vm, run_mode)); rustpython_vm::common::os::exit_code(exitcode) @@ -351,7 +356,7 @@ mod tests { use rustpython_vm::Interpreter; fn interpreter() -> Interpreter { - InterpreterConfig::new().init_stdlib().interpreter() + InterpreterBuilder::new().init_stdlib().interpreter() } #[test] diff --git a/src/main.rs b/src/main.rs index e88ea40f3df..3953b9dacfe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,10 @@ +use rustpython::{InterpreterBuilder, InterpreterBuilderExt}; + pub fn main() -> std::process::ExitCode { - rustpython::run(|_vm| {}) + let mut config = InterpreterBuilder::new(); + #[cfg(feature = "stdlib")] + { + config = config.init_stdlib(); + } + rustpython::run(config) }