From 34180d824838831de8174a913f00020a61118f40 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 11 Feb 2026 18:35:38 +0900 Subject: [PATCH 1/2] Fix annotation scope, deadlock, MRO, and HEAPTYPE issues Annotation scope: - Remove module-level annotation re-scan that created phantom sub_tables, breaking annotation closure for comprehensions - Add async comprehension check in symbol table with is_in_async_context(); annotation/type-params scopes are always non-async - Save/restore CompileContext in enter/exit_annotation_scope to reset in_async_scope Deadlock prevention: - Fix TypeVar/ParamSpec/TypeVarTuple __default__ and evaluate_default by cloning lock contents before acquiring a second lock or calling Python Other fixes: - Add HEAPTYPE flag to Generic for correct pickle behavior - Guard heaptype_ext access in name_inner/set___name__/ set___qualname__ with safe checks instead of unwrap - Fix MRO error message to include base class names - Add "format" to varnames in TypeAlias annotation scopes - Fix single-element tuple repr to include trailing comma --- crates/codegen/src/compile.rs | 57 +++++++++++++++++---- crates/codegen/src/symboltable.rs | 43 ++++++++++++---- crates/vm/src/builtins/type.rs | 42 +++++++++------- crates/vm/src/stdlib/typevar.rs | 83 +++++++++++++++++-------------- crates/vm/src/stdlib/typing.rs | 7 ++- 5 files changed, 156 insertions(+), 76 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index c804dc0ca26..a2f32448ff2 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -1226,21 +1226,35 @@ impl Compiler { } /// Exit annotation scope - similar to exit_scope but restores annotation_block to parent - fn exit_annotation_scope(&mut self) -> CodeObject { + fn exit_annotation_scope(&mut self, saved_ctx: CompileContext) -> CodeObject { self.pop_annotation_symbol_table(); + self.ctx = saved_ctx; let pop = self.code_stack.pop(); let stack_top = compiler_unwrap_option(self, pop); unwrap_internal(self, stack_top.finalize_code(&self.opts)) } - /// Enter annotation scope using the symbol table's annotation_block - /// Returns false if no annotation_block exists - fn enter_annotation_scope(&mut self, _func_name: &str) -> CompileResult { + /// Enter annotation scope using the symbol table's annotation_block. + /// Returns None if no annotation_block exists. + /// On success, returns the saved CompileContext to pass to exit_annotation_scope. + fn enter_annotation_scope( + &mut self, + _func_name: &str, + ) -> CompileResult> { if !self.push_annotation_symbol_table() { - return Ok(false); + return Ok(None); } + // Annotation scopes are never async (even inside async functions) + let saved_ctx = self.ctx; + self.ctx = CompileContext { + loop_data: None, + in_class: saved_ctx.in_class, + func: FunctionContext::Function, + in_async_scope: false, + }; + let key = self.symbol_table_stack.len() - 1; let lineno = self.get_source_line_number().get(); self.enter_scope( @@ -1261,7 +1275,7 @@ impl Compiler { // VALUE_WITH_FAKE_GLOBALS = 2 (from annotationlib.Format) self.emit_format_validation()?; - Ok(true) + Ok(Some(saved_ctx)) } /// Emit format parameter validation for annotation scope @@ -2477,6 +2491,10 @@ impl Compiler { // Evaluator takes a positional-only format parameter self.current_code_info().metadata.argcount = 1; self.current_code_info().metadata.posonlyargcount = 1; + self.current_code_info() + .metadata + .varnames + .insert("format".to_owned()); self.emit_format_validation()?; self.compile_expression(value)?; emit!(self, Instruction::ReturnValue); @@ -2514,6 +2532,10 @@ impl Compiler { // Evaluator takes a positional-only format parameter self.current_code_info().metadata.argcount = 1; self.current_code_info().metadata.posonlyargcount = 1; + self.current_code_info() + .metadata + .varnames + .insert("format".to_owned()); self.emit_format_validation()?; let prev_ctx = self.ctx; @@ -2659,6 +2681,10 @@ impl Compiler { // Evaluator takes a positional-only format parameter self.current_code_info().metadata.argcount = 1; self.current_code_info().metadata.posonlyargcount = 1; + self.current_code_info() + .metadata + .varnames + .insert("format".to_owned()); self.emit_format_validation()?; @@ -3787,10 +3813,10 @@ impl Compiler { parameters: &ast::Parameters, returns: Option<&ast::Expr>, ) -> CompileResult { - // Try to enter annotation scope - returns false if no annotation_block exists - if !self.enter_annotation_scope(func_name)? { + // Try to enter annotation scope - returns None if no annotation_block exists + let Some(saved_ctx) = self.enter_annotation_scope(func_name)? else { return Ok(false); - } + }; // Count annotations let parameters_iter = core::iter::empty() @@ -3842,7 +3868,7 @@ impl Compiler { emit!(self, Instruction::ReturnValue); // Exit the annotation scope and get the code object - let annotate_code = self.exit_annotation_scope(); + let annotate_code = self.exit_annotation_scope(saved_ctx); // Make a closure from the code object self.make_closure(annotate_code, bytecode::MakeFunctionFlags::empty())?; @@ -3935,6 +3961,15 @@ impl Compiler { return Ok(false); } + // Annotation scopes are never async (even inside async functions) + let saved_ctx = self.ctx; + self.ctx = CompileContext { + loop_data: None, + in_class: saved_ctx.in_class, + func: FunctionContext::Function, + in_async_scope: false, + }; + // Enter annotation scope for code generation let key = self.symbol_table_stack.len() - 1; let lineno = self.get_source_line_number().get(); @@ -4031,6 +4066,8 @@ impl Compiler { .last_mut() .expect("no module symbol table") .annotation_block = Some(Box::new(annotation_table)); + // Restore context + self.ctx = saved_ctx; // Exit code scope let pop = self.code_stack.pop(); let annotate_code = unwrap_internal( diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index 4afd4cc75da..62dd6361f50 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -1026,6 +1026,26 @@ impl SymbolTableBuilder { .insert(SymbolFlags::REFERENCED | SymbolFlags::FREE_CLASS); } + /// Walk up the scope chain to determine if we're inside an async function. + /// Annotation and TypeParams scopes act as async barriers (always non-async). + /// Comprehension scopes are transparent (inherit parent's async context). + fn is_in_async_context(&self) -> bool { + for table in self.tables.iter().rev() { + match table.typ { + CompilerScope::AsyncFunction => return true, + CompilerScope::Function + | CompilerScope::Lambda + | CompilerScope::Class + | CompilerScope::Module + | CompilerScope::Annotation + | CompilerScope::TypeParams => return false, + // Comprehension inherits parent's async context + CompilerScope::Comprehension => continue, + } + } + false + } + fn line_index_start(&self, range: TextRange) -> u32 { self.source_file .to_source_code() @@ -1128,15 +1148,6 @@ impl SymbolTableBuilder { self.leave_annotation_scope(); - // Module scope: re-scan to register symbols (builtins like str, int) - // Class scope: do NOT re-scan to preserve class-local symbol resolution - if matches!(current_scope, Some(CompilerScope::Module)) { - let was_in_annotation = self.in_annotation; - self.in_annotation = true; - let _ = self.scan_expression(annotation, ExpressionContext::Load); - self.in_annotation = was_in_annotation; - } - result } @@ -1939,6 +1950,20 @@ impl SymbolTableBuilder { range: TextRange, is_generator: bool, ) -> SymbolTableResult { + // Check for async comprehension outside async function + // (list/set/dict comprehensions only, not generator expressions) + let has_async_gen = generators.iter().any(|g| g.is_async); + if has_async_gen && !is_generator && !self.is_in_async_context() { + return Err(SymbolTableError { + error: "asynchronous comprehension outside of an asynchronous function".to_owned(), + location: Some( + self.source_file + .to_source_code() + .source_location(range.start(), PositionEncoding::Utf8), + ), + }); + } + // Comprehensions are compiled as functions, so create a scope for them: self.enter_scope( scope_name, diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index bdbbc367e18..49257117484 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -605,10 +605,10 @@ impl PyType { static_f: impl FnOnce(&'static str) -> R, heap_f: impl FnOnce(&'a HeapTypeExt) -> R, ) -> R { - if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { - static_f(self.slots.name) + if let Some(ref ext) = self.heaptype_ext { + heap_f(ext) } else { - heap_f(self.heaptype_ext.as_ref().unwrap()) + static_f(self.slots.name) } } @@ -849,13 +849,7 @@ impl PyType { #[pygetset(setter)] fn set___qualname__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { - // TODO: we should replace heaptype flag check to immutable flag check - if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { - return Err(vm.new_type_error(format!( - "cannot set '__qualname__' attribute of immutable type '{}'", - self.name() - ))); - }; + self.check_set_special_type_attr(identifier!(vm, __qualname__), vm)?; let value = value.ok_or_else(|| { vm.new_type_error(format!( "cannot delete '__qualname__' attribute of immutable type '{}'", @@ -865,10 +859,12 @@ impl PyType { let str_value = downcast_qualname(value, vm)?; - let heap_type = self - .heaptype_ext - .as_ref() - .expect("HEAPTYPE should have heaptype_ext"); + let heap_type = self.heaptype_ext.as_ref().ok_or_else(|| { + vm.new_type_error(format!( + "cannot set '__qualname__' attribute of immutable type '{}'", + self.name() + )) + })?; // Use std::mem::replace to swap the new value in and get the old value out, // then drop the old value after releasing the lock @@ -1160,10 +1156,17 @@ impl PyType { } name.ensure_valid_utf8(vm)?; + let heap_type = self.heaptype_ext.as_ref().ok_or_else(|| { + vm.new_type_error(format!( + "cannot set '__name__' attribute of immutable type '{}'", + self.slot_name() + )) + })?; + // Use std::mem::replace to swap the new value in and get the old value out, - // then drop the old value after releasing the lock (similar to CPython's Py_SETREF) + // then drop the old value after releasing the lock let _old_name = { - let mut name_guard = self.heaptype_ext.as_ref().unwrap().name.write(); + let mut name_guard = heap_type.name.write(); core::mem::replace(&mut *name_guard, name) }; // old_name is dropped here, outside the lock scope @@ -2129,9 +2132,10 @@ fn linearise_mro(mut bases: Vec>) -> Result, Strin // We start at index 1 to skip direct bases. // This will not catch duplicate bases, but such a thing is already tested for. if later_mro[1..].iter().any(|cls| cls.is(base)) { - return Err( - "Unable to find mro order which keeps local precedence ordering".to_owned(), - ); + return Err(format!( + "Cannot create a consistent method resolution order (MRO) for bases {}", + bases.iter().map(|x| x.first().unwrap()).format(", ") + )); } } } diff --git a/crates/vm/src/stdlib/typevar.rs b/crates/vm/src/stdlib/typevar.rs index 9645f5a86ef..33513b25027 100644 --- a/crates/vm/src/stdlib/typevar.rs +++ b/crates/vm/src/stdlib/typevar.rs @@ -138,14 +138,17 @@ pub(crate) mod typevar { #[pygetset] fn __default__(&self, vm: &VirtualMachine) -> PyResult { - let mut default_value = self.default_value.lock(); - if !default_value.is(&vm.ctx.typing_no_default) { - return Ok(default_value.clone()); + { + let default_value = self.default_value.lock(); + if !default_value.is(&vm.ctx.typing_no_default) { + return Ok(default_value.clone()); + } } - let evaluate_default = self.evaluate_default.lock(); - if !vm.is_none(&evaluate_default) { - *default_value = evaluate_default.call((1i32,), vm)?; - Ok(default_value.clone()) + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + let result = evaluator.call((1i32,), vm)?; + *self.default_value.lock() = result.clone(); + Ok(result) } else { Ok(vm.ctx.typing_no_default.clone().into()) } @@ -177,13 +180,13 @@ pub(crate) mod typevar { #[pygetset] fn evaluate_default(&self, vm: &VirtualMachine) -> PyResult { - let evaluate_default = self.evaluate_default.lock(); - if !vm.is_none(&evaluate_default) { - return Ok(evaluate_default.clone()); + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + return Ok(evaluator); } - let default_value = self.default_value.lock(); + let default_value = self.default_value.lock().clone(); if !default_value.is(&vm.ctx.typing_no_default) { - return Ok(const_evaluator_alloc(default_value.clone(), vm)); + return Ok(const_evaluator_alloc(default_value, vm)); } Ok(vm.ctx.none()) } @@ -500,14 +503,17 @@ pub(crate) mod typevar { #[pygetset] fn __default__(&self, vm: &VirtualMachine) -> PyResult { - let mut default_value = self.default_value.lock(); - if !default_value.is(&vm.ctx.typing_no_default) { - return Ok(default_value.clone()); + { + let default_value = self.default_value.lock(); + if !default_value.is(&vm.ctx.typing_no_default) { + return Ok(default_value.clone()); + } } - let evaluate_default = self.evaluate_default.lock(); - if !vm.is_none(&evaluate_default) { - *default_value = evaluate_default.call((1i32,), vm)?; - Ok(default_value.clone()) + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + let result = evaluator.call((1i32,), vm)?; + *self.default_value.lock() = result.clone(); + Ok(result) } else { Ok(vm.ctx.typing_no_default.clone().into()) } @@ -515,13 +521,13 @@ pub(crate) mod typevar { #[pygetset] fn evaluate_default(&self, vm: &VirtualMachine) -> PyResult { - let evaluate_default = self.evaluate_default.lock(); - if !vm.is_none(&evaluate_default) { - return Ok(evaluate_default.clone()); + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + return Ok(evaluator); } - let default_value = self.default_value.lock(); + let default_value = self.default_value.lock().clone(); if !default_value.is(&vm.ctx.typing_no_default) { - return Ok(const_evaluator_alloc(default_value.clone(), vm)); + return Ok(const_evaluator_alloc(default_value, vm)); } Ok(vm.ctx.none()) } @@ -695,14 +701,17 @@ pub(crate) mod typevar { #[pygetset] fn __default__(&self, vm: &VirtualMachine) -> PyResult { - let mut default_value = self.default_value.lock(); - if !default_value.is(&vm.ctx.typing_no_default) { - return Ok(default_value.clone()); + { + let default_value = self.default_value.lock(); + if !default_value.is(&vm.ctx.typing_no_default) { + return Ok(default_value.clone()); + } } - let evaluate_default = self.evaluate_default.lock(); - if !vm.is_none(&evaluate_default) { - *default_value = evaluate_default.call((1i32,), vm)?; - Ok(default_value.clone()) + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + let result = evaluator.call((1i32,), vm)?; + *self.default_value.lock() = result.clone(); + Ok(result) } else { Ok(vm.ctx.typing_no_default.clone().into()) } @@ -710,13 +719,13 @@ pub(crate) mod typevar { #[pygetset] fn evaluate_default(&self, vm: &VirtualMachine) -> PyResult { - let evaluate_default = self.evaluate_default.lock(); - if !vm.is_none(&evaluate_default) { - return Ok(evaluate_default.clone()); + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + return Ok(evaluator); } - let default_value = self.default_value.lock(); + let default_value = self.default_value.lock().clone(); if !default_value.is(&vm.ctx.typing_no_default) { - return Ok(const_evaluator_alloc(default_value.clone(), vm)); + return Ok(const_evaluator_alloc(default_value, vm)); } Ok(vm.ctx.none()) } @@ -1034,7 +1043,7 @@ pub(crate) mod typevar { #[allow(dead_code)] pub struct Generic; - #[pyclass(flags(BASETYPE))] + #[pyclass(flags(BASETYPE, HEAPTYPE))] impl Generic { #[pyattr] fn __slots__(ctx: &Context) -> PyTupleRef { diff --git a/crates/vm/src/stdlib/typing.rs b/crates/vm/src/stdlib/typing.rs index 9ae33cea7ee..6887106b4b5 100644 --- a/crates/vm/src/stdlib/typing.rs +++ b/crates/vm/src/stdlib/typing.rs @@ -162,7 +162,12 @@ pub(crate) mod decl { for item in tuple.iter() { parts.push(typing_type_repr(item, vm)?); } - Ok(vm.ctx.new_str(format!("({})", parts.join(", "))).into()) + let inner = if parts.len() == 1 { + format!("{},", parts[0]) + } else { + parts.join(", ") + }; + Ok(vm.ctx.new_str(format!("({})", inner)).into()) } else { Ok(vm.ctx.new_str(typing_type_repr(value, vm)?).into()) } From ba6d7b5404810accf967bd8a94e327382ae8c744 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 11 Feb 2026 18:35:54 +0900 Subject: [PATCH 2/2] Unmark failing markers --- Lib/ctypes/__init__.py | 1 + Lib/test/test_dataclasses/__init__.py | 1 - Lib/test/test_genericalias.py | 2 -- Lib/test/test_type_annotations.py | 2 -- Lib/test/test_type_params.py | 2 -- Lib/test/test_typing.py | 2 -- 6 files changed, 1 insertion(+), 9 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 80651dc64ce..d67261c461f 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -163,6 +163,7 @@ def __repr__(self): return super().__repr__() except ValueError: return "%s()" % type(self).__name__ + __class_getitem__ = classmethod(_types.GenericAlias) _check_size(py_object, "P") class c_short(_SimpleCData): diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 4d237bdd1a8..12db84a1209 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -86,7 +86,6 @@ def test_field_recursive_repr(self): self.assertIn(",type=...,", repr_output) - @unittest.expectedFailure # TODO: RUSTPYTHON; recursive annotation type not shown as ... def test_recursive_annotation(self): class C: pass diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index cc0ca93e79b..4f5b10650ac 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -152,7 +152,6 @@ class BaseTest(unittest.TestCase): if Event is not None: generic_types.append(Event) - @unittest.expectedFailure # TODO: RUSTPYTHON; memoryview, Template, Interpolation, py_object not subscriptable def test_subscriptable(self): for t in self.generic_types: if t is None: @@ -507,7 +506,6 @@ def test_dir(self): with self.subTest(entry=entry): getattr(ga, entry) # must not raise `AttributeError` - @unittest.expectedFailure # TODO: RUSTPYTHON; memoryview, Template, Interpolation, py_object not subscriptable def test_weakref(self): for t in self.generic_types: if t is None: diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 4ed786cca3a..a7b87bb2ee0 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -413,7 +413,6 @@ class Nested: ... self.assertEqual(Outer.meth.__annotations__, {"x": Outer.Nested}) self.assertEqual(Outer.__annotations__, {"x": Outer.Nested}) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_no_exotic_expressions(self): preludes = [ "", @@ -431,7 +430,6 @@ def test_no_exotic_expressions(self): check_syntax_error(self, prelude + "def func(x: {y async for y in x}): ...", "asynchronous comprehension outside of an asynchronous function") check_syntax_error(self, prelude + "def func(x: {y: y async for y in x}): ...", "asynchronous comprehension outside of an asynchronous function") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_no_exotic_expressions_in_unevaluated_annotations(self): preludes = [ "", diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 269c325d642..753f485fa85 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -148,7 +148,6 @@ def test_disallowed_expressions(self): check_syntax_error(self, "def f[T: [(x := 3) for _ in range(2)]](): pass") check_syntax_error(self, "type T = [(x := 3) for _ in range(2)]") - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "\(MRO\) for bases object, Generic" does not match "Unable to find mro order which keeps local precedence ordering" def test_incorrect_mro_explicit_object(self): with self.assertRaisesRegex(TypeError, r"\(MRO\) for bases object, Generic"): class My[X](object): ... @@ -1215,7 +1214,6 @@ def test_pickling_functions(self): pickled = pickle.dumps(thing, protocol=proto) self.assertEqual(pickle.loads(pickled), thing) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_pickling_classes(self): things_to_test = [ Class1, diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 788da8f41e4..1038e8c1d1d 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4209,7 +4209,6 @@ class P(Protocol): Alias2 = typing.Union[P, typing.Iterable] self.assertEqual(Alias, Alias2) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Generic() takes no arguments def test_protocols_pickleable(self): global P, CP # pickle wants to reference the class by name T = TypeVar('T') @@ -5287,7 +5286,6 @@ def test_all_repr_eq_any(self): self.assertNotEqual(repr(base), '') self.assertEqual(base, base) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Generic() takes no arguments def test_pickle(self): global C # pickle wants to reference the class by name T = TypeVar('T')