From 5bfbb77a2832c4cd2a4e47de7983836b175000e4 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 14 Jul 2025 23:29:10 +0900 Subject: [PATCH 1/6] PyRef::into_non_null --- vm/src/object/core.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/vm/src/object/core.rs b/vm/src/object/core.rs index 5012855133a..53fc5b68c56 100644 --- a/vm/src/object/core.rs +++ b/vm/src/object/core.rs @@ -1018,11 +1018,23 @@ impl Clone for PyRef { } impl PyRef { + #[inline(always)] + pub(crate) const fn into_non_null(self) -> NonNull> { + let ptr = self.ptr; + std::mem::forget(self); + ptr + } + + #[inline(always)] + pub(crate) const unsafe fn from_non_null(ptr: NonNull>) -> Self { + Self { ptr } + } + + /// # Safety + /// The raw pointer must point to a valid `Py` object #[inline(always)] pub(crate) const unsafe fn from_raw(raw: *const Py) -> Self { - Self { - ptr: unsafe { NonNull::new_unchecked(raw as *mut _) }, - } + unsafe { Self::from_non_null(NonNull::new_unchecked(raw as *mut _)) } } /// Safety: payload type of `obj` must be `T` From 53d353bd55a31ea7c1c1b6aa27c5f01c52f1a311 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 14 Jul 2025 13:26:07 +0900 Subject: [PATCH 2/6] SetFunctionAttribute --- Lib/test/test_funcattrs.py | 2 - Lib/test/test_reprlib.py | 2 - Lib/test/test_typing.py | 2 - compiler/codegen/src/compile.rs | 355 +++++++++++++++++++--------- compiler/codegen/src/symboltable.rs | 1 + compiler/core/src/bytecode.rs | 18 +- jit/tests/common.rs | 46 +++- vm/src/builtins/function.rs | 36 ++- vm/src/frame.rs | 141 ++++++----- 9 files changed, 409 insertions(+), 194 deletions(-) diff --git a/Lib/test/test_funcattrs.py b/Lib/test/test_funcattrs.py index 5fd268fd902..3d5378092b4 100644 --- a/Lib/test/test_funcattrs.py +++ b/Lib/test/test_funcattrs.py @@ -176,8 +176,6 @@ def test___name__(self): self.assertEqual(self.fi.a.__name__, 'a') self.cannot_set_attr(self.fi.a, "__name__", 'a', AttributeError) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test___qualname__(self): # PEP 3155 self.assertEqual(self.b.__qualname__, 'FuncAttrsTest.setUp..b') diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py index 396be4b1044..738b48f5623 100644 --- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -176,8 +176,6 @@ def test_instance(self): self.assertTrue(s.endswith(">")) self.assertIn(s.find("..."), [12, 13]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lambda(self): r = repr(lambda x: x) self.assertTrue(r.startswith(". { Ok(()) } - fn enter_function( - &mut self, - name: &str, - parameters: &Parameters, - ) -> CompileResult { - let defaults: Vec<_> = std::iter::empty() - .chain(¶meters.posonlyargs) - .chain(¶meters.args) - .filter_map(|x| x.default.as_deref()) - .collect(); - let have_defaults = !defaults.is_empty(); - if have_defaults { - // Construct a tuple: - let size = defaults.len().to_u32(); - for element in &defaults { - self.compile_expression(element)?; - } - emit!(self, Instruction::BuildTuple { size }); - } - + fn enter_function(&mut self, name: &str, parameters: &Parameters) -> CompileResult<()> { // TODO: partition_in_place let mut kw_without_defaults = vec![]; let mut kw_with_defaults = vec![]; @@ -1513,31 +1494,6 @@ impl Compiler<'_> { } } - // let (kw_without_defaults, kw_with_defaults) = args.split_kwonlyargs(); - if !kw_with_defaults.is_empty() { - let default_kw_count = kw_with_defaults.len(); - for (arg, default) in kw_with_defaults.iter() { - self.emit_load_const(ConstantData::Str { - value: arg.name.as_str().into(), - }); - self.compile_expression(default)?; - } - emit!( - self, - Instruction::BuildMap { - size: default_kw_count.to_u32(), - } - ); - } - - let mut func_flags = bytecode::MakeFunctionFlags::empty(); - if have_defaults { - func_flags |= bytecode::MakeFunctionFlags::DEFAULTS; - } - if !kw_with_defaults.is_empty() { - func_flags |= bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS; - } - self.push_output( bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED, parameters.posonlyargs.len().to_u32(), @@ -1565,7 +1521,7 @@ impl Compiler<'_> { self.varname(name.name.as_str())?; } - Ok(func_flags) + Ok(()) } fn prepare_decorators(&mut self, decorator_list: &[Decorator]) -> CompileResult<()> { @@ -1869,7 +1825,57 @@ impl Compiler<'_> { self.push_symbol_table(); } - let mut func_flags = self.enter_function(name, parameters)?; + // Prepare defaults and kwdefaults before entering function + let defaults: Vec<_> = std::iter::empty() + .chain(¶meters.posonlyargs) + .chain(¶meters.args) + .filter_map(|x| x.default.as_deref()) + .collect(); + let have_defaults = !defaults.is_empty(); + + // Compile defaults before entering function scope + if have_defaults { + // Construct a tuple: + let size = defaults.len().to_u32(); + for element in &defaults { + self.compile_expression(element)?; + } + emit!(self, Instruction::BuildTuple { size }); + } + + // Prepare keyword-only defaults + let mut kw_with_defaults = vec![]; + for kwonlyarg in ¶meters.kwonlyargs { + if let Some(default) = &kwonlyarg.default { + kw_with_defaults.push((&kwonlyarg.parameter, default)); + } + } + + let have_kwdefaults = !kw_with_defaults.is_empty(); + if have_kwdefaults { + let default_kw_count = kw_with_defaults.len(); + for (arg, default) in kw_with_defaults.iter() { + self.emit_load_const(ConstantData::Str { + value: arg.name.as_str().into(), + }); + self.compile_expression(default)?; + } + emit!( + self, + Instruction::BuildMap { + size: default_kw_count.to_u32(), + } + ); + } + + self.enter_function(name, parameters)?; + let mut func_flags = bytecode::MakeFunctionFlags::empty(); + if have_defaults { + func_flags |= bytecode::MakeFunctionFlags::DEFAULTS; + } + if have_kwdefaults { + func_flags |= bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS; + } self.current_code_info() .flags .set(bytecode::CodeFlags::IS_COROUTINE, is_async); @@ -1888,7 +1894,7 @@ impl Compiler<'_> { }; // Set qualname using the new method - let qualname = self.set_qualname(); + self.set_qualname(); let (doc_str, body) = split_doc(body, &self.opts); @@ -1965,7 +1971,7 @@ impl Compiler<'_> { } // Create function with closure - self.make_closure(code, &qualname, func_flags)?; + self.make_closure(code, func_flags)?; if let Some(value) = doc_str { emit!(self, Instruction::Duplicate); @@ -1982,58 +1988,97 @@ impl Compiler<'_> { self.store_name(name) } + /// Determines if a variable should be CELL or FREE type + /// Equivalent to CPython's get_ref_type + fn get_ref_type(&self, name: &str) -> Result { + // Special handling for __class__ and __classdict__ in class scope + if self.ctx.in_class && (name == "__class__" || name == "__classdict__") { + return Ok(SymbolScope::Cell); + } + + let table = self.symbol_table_stack.last().unwrap(); + match table.lookup(name) { + Some(symbol) => match symbol.scope { + SymbolScope::Cell | SymbolScope::TypeParams => Ok(SymbolScope::Cell), + SymbolScope::Free => Ok(SymbolScope::Free), + _ if symbol.flags.contains(SymbolFlags::FREE_CLASS) => Ok(SymbolScope::Free), + _ => Err(CodegenErrorType::SyntaxError(format!( + "get_ref_type: invalid scope for '{name}'" + ))), + }, + None => Err(CodegenErrorType::SyntaxError(format!( + "get_ref_type: cannot find symbol '{name}'" + ))), + } + } + /// Loads closure variables if needed and creates a function object // = compiler_make_closure fn make_closure( &mut self, code: CodeObject, - qualname: &str, - mut flags: bytecode::MakeFunctionFlags, + flags: bytecode::MakeFunctionFlags, ) -> CompileResult<()> { // Handle free variables (closure) - if !code.freevars.is_empty() { + let has_freevars = !code.freevars.is_empty(); + if has_freevars { // Build closure tuple by loading free variables - for var in &code.freevars { - let table = self.symbol_table_stack.last().unwrap(); - let symbol = match table.lookup(var) { - Some(s) => s, - None => { - return Err(self.error(CodegenErrorType::SyntaxError(format!( - "compiler_make_closure: cannot find symbol '{var}'", - )))); - } - }; + for var in &*code.freevars { + // Special case: If a class contains a method with a + // free variable that has the same name as a method, + // the name will be considered free *and* local in the + // class. It should be handled by the closure, as + // well as by the normal name lookup logic. + + // Get reference type using our get_ref_type function + let ref_type = self.get_ref_type(var).map_err(|e| self.error(e))?; + + // Get parent code info let parent_code = self.code_stack.last().unwrap(); - let vars = match symbol.scope { - SymbolScope::Free => &parent_code.metadata.freevars, - SymbolScope::Cell => &parent_code.metadata.cellvars, - SymbolScope::TypeParams => &parent_code.metadata.cellvars, - _ if symbol.flags.contains(SymbolFlags::FREE_CLASS) => { - &parent_code.metadata.freevars + let cellvars_len = parent_code.metadata.cellvars.len(); + + // Look up the variable index based on reference type + let idx = match ref_type { + SymbolScope::Cell => { + match parent_code.metadata.cellvars.get_index_of(var) { + Some(i) => i, + None => { + // Try in freevars as fallback + match parent_code.metadata.freevars.get_index_of(var) { + Some(i) => i + cellvars_len, + None => { + return Err(self.error(CodegenErrorType::SyntaxError(format!( + "compiler_make_closure: cannot find '{var}' in parent vars", + )))); + } + } + } + } } - _ => { - return Err(self.error(CodegenErrorType::SyntaxError(format!( - "compiler_make_closure: invalid scope for '{var}'", - )))); + SymbolScope::Free => { + match parent_code.metadata.freevars.get_index_of(var) { + Some(i) => i + cellvars_len, + None => { + // Try in cellvars as fallback + match parent_code.metadata.cellvars.get_index_of(var) { + Some(i) => i, + None => { + return Err(self.error(CodegenErrorType::SyntaxError(format!( + "compiler_make_closure: cannot find '{var}' in parent vars", + )))); + } + } + } + } } - }; - - let idx = match vars.get_index_of(var) { - Some(i) => i, - None => { + _ => { return Err(self.error(CodegenErrorType::SyntaxError(format!( - "compiler_make_closure: cannot find '{var}' in parent vars", + "compiler_make_closure: unexpected ref_type {ref_type:?} for '{var}'", )))); } }; - let idx = if let SymbolScope::Free = symbol.scope { - idx + parent_code.metadata.cellvars.len() - } else { - idx - }; - emit!(self, Instruction::LoadClosure(idx.to_u32())); } @@ -2044,22 +2089,76 @@ impl Compiler<'_> { size: code.freevars.len().to_u32(), } ); - - flags |= bytecode::MakeFunctionFlags::CLOSURE; } - // Load code object + // CPython 3.13 style: load code object and create function self.emit_load_const(ConstantData::Code { code: Box::new(code), }); - // Load qualified name - self.emit_load_const(ConstantData::Str { - value: qualname.into(), - }); + // Create function with no flags + emit!( + self, + Instruction::MakeFunction(bytecode::MakeFunctionFlags::empty()) + ); + + // Now set attributes one by one using SET_FUNCTION_ATTRIBUTE + // Note: The order matters! Values must be on stack before calling SET_FUNCTION_ATTRIBUTE + + // Set closure if needed + if has_freevars { + // Closure tuple is already on stack + emit!( + self, + Instruction::SetFunctionAttribute { + attr: bytecode::MakeFunctionFlags::CLOSURE + } + ); + } + + // Set annotations if present + if flags.contains(bytecode::MakeFunctionFlags::ANNOTATIONS) { + // Annotations dict is already on stack + emit!( + self, + Instruction::SetFunctionAttribute { + attr: bytecode::MakeFunctionFlags::ANNOTATIONS + } + ); + } + + // Set kwdefaults if present + if flags.contains(bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS) { + // kwdefaults dict is already on stack + emit!( + self, + Instruction::SetFunctionAttribute { + attr: bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS + } + ); + } - // Make function with proper flags - emit!(self, Instruction::MakeFunction(flags)); + // Set defaults if present + if flags.contains(bytecode::MakeFunctionFlags::DEFAULTS) { + // defaults tuple is already on stack + emit!( + self, + Instruction::SetFunctionAttribute { + attr: bytecode::MakeFunctionFlags::DEFAULTS + } + ); + } + + // Set type_params if present + if flags.contains(bytecode::MakeFunctionFlags::TYPE_PARAMS) { + // type_params tuple is already on stack + emit!( + self, + Instruction::SetFunctionAttribute { + attr: bytecode::MakeFunctionFlags::TYPE_PARAMS + } + ); + } Ok(()) } @@ -2262,7 +2361,7 @@ impl Compiler<'_> { func_flags |= bytecode::MakeFunctionFlags::TYPE_PARAMS; // Create class function with closure - self.make_closure(class_code, name, func_flags)?; + self.make_closure(class_code, func_flags)?; self.emit_load_const(ConstantData::Str { value: name.into() }); // Compile original bases @@ -2311,19 +2410,14 @@ impl Compiler<'_> { let type_params_code = self.exit_scope(); // Execute the type params function - let type_params_name = format!(""); - self.make_closure( - type_params_code, - &type_params_name, - bytecode::MakeFunctionFlags::empty(), - )?; + self.make_closure(type_params_code, bytecode::MakeFunctionFlags::empty())?; emit!(self, Instruction::CallFunctionPositional { nargs: 0 }); } else { // Non-generic class: standard path emit!(self, Instruction::LoadBuildClass); // Create class function with closure - self.make_closure(class_code, name, bytecode::MakeFunctionFlags::empty())?; + self.make_closure(class_code, bytecode::MakeFunctionFlags::empty())?; self.emit_load_const(ConstantData::Str { value: name.into() }); let call = if let Some(arguments) = arguments { @@ -4033,10 +4127,59 @@ impl Compiler<'_> { parameters, body, .. }) => { let prev_ctx = self.ctx; - let name = "".to_owned(); - let func_flags = self - .enter_function(&name, parameters.as_deref().unwrap_or(&Default::default()))?; + let default_params = Default::default(); + let params = parameters.as_deref().unwrap_or(&default_params); + + // Prepare defaults before entering function + let defaults: Vec<_> = std::iter::empty() + .chain(¶ms.posonlyargs) + .chain(¶ms.args) + .filter_map(|x| x.default.as_deref()) + .collect(); + let have_defaults = !defaults.is_empty(); + + if have_defaults { + let size = defaults.len().to_u32(); + for element in &defaults { + self.compile_expression(element)?; + } + emit!(self, Instruction::BuildTuple { size }); + } + + // Prepare keyword-only defaults + let mut kw_with_defaults = vec![]; + for kwonlyarg in ¶ms.kwonlyargs { + if let Some(default) = &kwonlyarg.default { + kw_with_defaults.push((&kwonlyarg.parameter, default)); + } + } + + let have_kwdefaults = !kw_with_defaults.is_empty(); + if have_kwdefaults { + let default_kw_count = kw_with_defaults.len(); + for (arg, default) in kw_with_defaults.iter() { + self.emit_load_const(ConstantData::Str { + value: arg.name.as_str().into(), + }); + self.compile_expression(default)?; + } + emit!( + self, + Instruction::BuildMap { + size: default_kw_count.to_u32(), + } + ); + } + + self.enter_function(&name, params)?; + let mut func_flags = bytecode::MakeFunctionFlags::empty(); + if have_defaults { + func_flags |= bytecode::MakeFunctionFlags::DEFAULTS; + } + if have_kwdefaults { + func_flags |= bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS; + } // Set qualname for lambda self.set_qualname(); @@ -4057,7 +4200,7 @@ impl Compiler<'_> { let code = self.exit_scope(); // Create lambda function with closure - self.make_closure(code, &name, func_flags)?; + self.make_closure(code, func_flags)?; self.ctx = prev_ctx; } @@ -4602,7 +4745,7 @@ impl Compiler<'_> { self.ctx = prev_ctx; // Create comprehension function with closure - self.make_closure(code, name, bytecode::MakeFunctionFlags::empty())?; + self.make_closure(code, bytecode::MakeFunctionFlags::empty())?; // Evaluate iterated item: self.compile_expression(&generators[0].iter)?; diff --git a/compiler/codegen/src/symboltable.rs b/compiler/codegen/src/symboltable.rs index 16a65bca116..7f7355bd734 100644 --- a/compiler/codegen/src/symboltable.rs +++ b/compiler/codegen/src/symboltable.rs @@ -127,6 +127,7 @@ pub enum SymbolScope { GlobalImplicit, Free, Cell, + // TODO: wrong place. not a symbol scope, but a COMPILER_SCOPE_TYPEPARAMS TypeParams, } diff --git a/compiler/core/src/bytecode.rs b/compiler/core/src/bytecode.rs index 3e74fe62738..2275bb6b97a 100644 --- a/compiler/core/src/bytecode.rs +++ b/compiler/core/src/bytecode.rs @@ -529,6 +529,9 @@ pub enum Instruction { target: Arg