diff --git a/.cspell.json b/.cspell.json index a7f814a340b..0a93ac35cd5 100644 --- a/.cspell.json +++ b/.cspell.json @@ -121,6 +121,7 @@ "sysmodule", "tracebacks", "typealiases", + "typevartuples", "unhashable", "uninit", "unraisable", diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 46dbdedd435..ef318dfc7e1 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -85,7 +85,7 @@ def test_traceback(self): self.assertIn('raise RuntimeError(123) # some comment', f1.getvalue()) - @unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON flaky EOFError') + @unittest.skip('TODO: RUSTPYTHON flaky EOFError') @hashlib_helper.requires_hashdigest('md5') def test_ressources_gced_in_workers(self): # Ensure that argument for a job are correctly gc-ed after the job diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 753f485fa85..af26a6a0a01 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -243,7 +243,6 @@ def func[A, B](a: dict[A, B]): A, B = func.__type_params__ self.assertEqual(func.__annotations__["a"], dict[A, B]) - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: the symbol 'list' must be present in the symbol table def test_function_access_02(self): code = """ def func[A](a = list[A]()): @@ -866,7 +865,6 @@ class Inner[T]: self.assertEqual(Outer.Inner._Inner__x, "inner") self.assertEqual(Outer._Outer__after, "after") - @unittest.expectedFailure # TODO: RUSTPYTHON; NameError: name '_Derived__Base' is not defined def test_no_mangling_in_bases(self): ns = run_code(""" class __Base: @@ -880,7 +878,6 @@ class Derived[T](__Base, __kwarg=1): self.assertEqual(Derived.__bases__, (ns["__Base"], Generic)) self.assertEqual(Derived.kwargs, {"__kwarg": 1}) - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: the symbol '_Y__X' must be present in the symbol table def test_no_mangling_in_nested_scopes(self): ns = run_code(""" from test.test_type_params import make_base @@ -911,7 +908,6 @@ class Y[T: __X]( base3 = Y.__bases__[3] self.assertEqual(list(base3.__arg__), [ns["__X"]]) - @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: the symbol '_Foo__T' must be present in the symbol table def test_type_params_are_mangled(self): ns = run_code(""" from test.test_type_params import make_base diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 7ec64ea5ccb..448d16f1f4a 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -10914,7 +10914,6 @@ class TypeIterationTests(BaseTestCase): Annotated[T, ''], ) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_cannot_iterate(self): expected_error_regex = "object is not iterable" for test_type in self._UNITERABLE_TYPES: diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 28d47cb0424..c13ec694a52 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -1852,7 +1852,8 @@ impl Compiler { .code_stack .last() .and_then(|info| info.private.as_deref()); - symboltable::mangle_name(private, name) + let mangled_names = self.current_symbol_table().mangled_names.as_ref(); + symboltable::maybe_mangle_name(private, mangled_names, name) } // = compiler_nameop diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index 62dd6361f50..22bea20fa6c 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -68,6 +68,11 @@ pub struct SymbolTable { /// Whether `from __future__ import annotations` is active pub future_annotations: bool, + + /// Names of type parameters that should still be mangled in type param scopes. + /// When Some, only names in this set are mangled; other names are left unmangled. + /// Set on type param blocks for generic classes; inherited by non-class child scopes. + pub mangled_names: Option>, } impl SymbolTable { @@ -88,6 +93,7 @@ impl SymbolTable { annotation_block: None, has_conditional_annotations: false, future_annotations: false, + mangled_names: None, } } @@ -905,14 +911,26 @@ impl SymbolTableBuilder { ) }) .unwrap_or(false); - let table = SymbolTable::new(name.to_owned(), typ, line_number, is_nested); + // Inherit mangled_names from parent for non-class scopes + let inherited_mangled_names = self + .tables + .last() + .and_then(|t| t.mangled_names.clone()) + .filter(|_| typ != CompilerScope::Class); + let mut table = SymbolTable::new(name.to_owned(), typ, line_number, is_nested); + table.mangled_names = inherited_mangled_names; self.tables.push(table); // Save parent's varnames and start fresh for the new scope self.varnames_stack .push(core::mem::take(&mut self.current_varnames)); } - fn enter_type_param_block(&mut self, name: &str, line_number: u32) -> SymbolTableResult { + fn enter_type_param_block( + &mut self, + name: &str, + line_number: u32, + for_class: bool, + ) -> SymbolTableResult { // Check if we're in a class scope let in_class = self .tables @@ -921,16 +939,21 @@ impl SymbolTableBuilder { self.enter_scope(name, CompilerScope::TypeParams, line_number); - // If we're in a class, mark that this type param scope can see the class scope + // Set properties on the newly created type param scope if let Some(table) = self.tables.last_mut() { table.can_see_class_scope = in_class; - - // Add __classdict__ as a USE symbol in type param scope if in class - if in_class { - self.register_name("__classdict__", SymbolUsage::Used, TextRange::default())?; + // For generic classes, create mangled_names set so that only + // type parameter names get mangled (not bases or other expressions) + if for_class { + table.mangled_names = Some(IndexSet::default()); } } + // Add __classdict__ as a USE symbol in type param scope if in class + if in_class { + self.register_name("__classdict__", SymbolUsage::Used, TextRange::default())?; + } + // Register .type_params as a SET symbol (it will be converted to cell variable later) self.register_name(".type_params", SymbolUsage::Assigned, TextRange::default())?; @@ -1202,12 +1225,20 @@ impl SymbolTableBuilder { None }; + // For generic functions, scan defaults before entering type_param_block + // (defaults are evaluated in the enclosing scope, not the type param scope) + let has_type_params = type_params.is_some(); + if has_type_params { + self.scan_parameter_defaults(parameters)?; + } + // For generic functions, enter type_param block FIRST so that // annotation scopes are nested inside and can see type parameters. if let Some(type_params) = type_params { self.enter_type_param_block( &format!("", name.as_str()), self.line_index_start(type_params.range), + false, )?; self.scan_type_params(type_params)?; } @@ -1223,6 +1254,7 @@ impl SymbolTableBuilder { self.line_index_start(*range), has_return_annotation, *is_async, + has_type_params, // skip_defaults: already scanned above )?; self.scan_statements(body)?; self.leave_scope(); @@ -1244,11 +1276,16 @@ impl SymbolTableBuilder { range, node_index: _, }) => { + // Save class_name for the entire ClassDef processing + let prev_class = self.class_name.take(); if let Some(type_params) = type_params { self.enter_type_param_block( &format!("", name.as_str()), self.line_index_start(type_params.range), + true, // for_class: enable selective mangling )?; + // Set class_name for mangling in type param scope + self.class_name = Some(name.to_string()); self.scan_type_params(type_params)?; } self.enter_scope( @@ -1257,10 +1294,9 @@ impl SymbolTableBuilder { self.line_index_start(*range), ); // Reset in_conditional_block for new class scope - // (each scope has its own conditional context) let saved_in_conditional = self.in_conditional_block; self.in_conditional_block = false; - let prev_class = self.class_name.replace(name.to_string()); + self.class_name = Some(name.to_string()); self.register_name("__module__", SymbolUsage::Assigned, *range)?; self.register_name("__qualname__", SymbolUsage::Assigned, *range)?; self.register_name("__doc__", SymbolUsage::Assigned, *range)?; @@ -1268,7 +1304,13 @@ impl SymbolTableBuilder { self.scan_statements(body)?; self.leave_scope(); self.in_conditional_block = saved_in_conditional; - self.class_name = prev_class; + // For non-generic classes, restore class_name before base scanning. + // Bases are evaluated in the enclosing scope, not the class scope. + // For generic classes, bases are scanned within the type_param scope + // where class_name is already correctly set. + if type_params.is_none() { + self.class_name = prev_class.clone(); + } if let Some(arguments) = arguments { self.scan_expressions(&arguments.args, ExpressionContext::Load)?; for keyword in &arguments.keywords { @@ -1278,6 +1320,8 @@ impl SymbolTableBuilder { if type_params.is_some() { self.leave_scope(); } + // Restore class_name after all ClassDef processing + self.class_name = prev_class; self.scan_decorators(decorator_list, ExpressionContext::Load)?; self.register_ident(name, SymbolUsage::Assigned)?; } @@ -1506,6 +1550,7 @@ impl SymbolTableBuilder { self.enter_type_param_block( "TypeAlias", self.line_index_start(type_params.range), + false, )?; self.scan_type_params(type_params)?; } @@ -1833,6 +1878,7 @@ impl SymbolTableBuilder { self.line_index_start(expression.range()), false, // lambdas have no return annotation false, // lambdas are never async + false, // don't skip defaults )?; } else { self.enter_scope( @@ -2233,6 +2279,20 @@ impl SymbolTableBuilder { Ok(()) } + /// Scan default parameter values (evaluated in the enclosing scope) + fn scan_parameter_defaults(&mut self, parameters: &ast::Parameters) -> SymbolTableResult { + for default in parameters + .posonlyargs + .iter() + .chain(parameters.args.iter()) + .chain(parameters.kwonlyargs.iter()) + .filter_map(|arg| arg.default.as_ref()) + { + self.scan_expression(default, ExpressionContext::Load)?; + } + Ok(()) + } + fn enter_scope_with_parameters( &mut self, name: &str, @@ -2240,16 +2300,11 @@ impl SymbolTableBuilder { line_number: u32, has_return_annotation: bool, is_async: bool, + skip_defaults: bool, ) -> SymbolTableResult { - // Evaluate eventual default parameters: - for default in parameters - .posonlyargs - .iter() - .chain(parameters.args.iter()) - .chain(parameters.kwonlyargs.iter()) - .filter_map(|arg| arg.default.as_ref()) - { - self.scan_expression(default, ExpressionContext::Load)?; // not ExprContext? + // Evaluate eventual default parameters (unless already scanned before type_param_block): + if !skip_defaults { + self.scan_parameter_defaults(parameters)?; } // Annotations are scanned in outer scope: @@ -2381,7 +2436,18 @@ impl SymbolTableBuilder { let scope_depth = self.tables.len(); let table = self.tables.last_mut().unwrap(); - let name = mangle_name(self.class_name.as_deref(), name); + // Add type param names to mangled_names set for selective mangling + if matches!(role, SymbolUsage::TypeParam) + && let Some(ref mut set) = table.mangled_names + { + set.insert(name.to_owned()); + } + + let name = maybe_mangle_name( + self.class_name.as_deref(), + table.mangled_names.as_ref(), + name, + ); // Some checks for the symbol that present on this scope level: let symbol = if let Some(symbol) = table.symbols.get_mut(name.as_ref()) { let flags = &symbol.flags; @@ -2574,11 +2640,27 @@ pub(crate) fn mangle_name<'a>(class_name: Option<&str>, name: &'a str) -> Cow<'a if !name.starts_with("__") || name.ends_with("__") || name.contains('.') { return name.into(); } - // strip leading underscore - let class_name = class_name.strip_prefix(|c| c == '_').unwrap_or(class_name); + // Strip leading underscores from class name + let class_name = class_name.trim_start_matches('_'); let mut ret = String::with_capacity(1 + class_name.len() + name.len()); ret.push('_'); ret.push_str(class_name); ret.push_str(name); ret.into() } + +/// Selective mangling for type parameter scopes around generic classes. +/// If `mangled_names` is Some, only mangle names that are in the set; +/// other names are left unmangled. +pub(crate) fn maybe_mangle_name<'a>( + class_name: Option<&str>, + mangled_names: Option<&IndexSet>, + name: &'a str, +) -> Cow<'a, str> { + if let Some(set) = mangled_names + && !set.contains(name) + { + return name.into(); + } + mangle_name(class_name, name) +} diff --git a/crates/vm/src/builtins/genericalias.rs b/crates/vm/src/builtins/genericalias.rs index f3759cb20bc..8aabca1ae10 100644 --- a/crates/vm/src/builtins/genericalias.rs +++ b/crates/vm/src/builtins/genericalias.rs @@ -720,24 +720,22 @@ impl crate::types::IterNext for PyGenericAliasIterator { } } -/// Creates a GenericAlias from type parameters, equivalent to CPython's _Py_subscript_generic -/// This is used for PEP 695 classes to create Generic[T] from type parameters +/// Creates a GenericAlias from type parameters, equivalent to _Py_subscript_generic. +/// This is used for PEP 695 classes to create Generic[T] from type parameters. // _Py_subscript_generic pub fn subscript_generic(type_params: PyObjectRef, vm: &VirtualMachine) -> PyResult { - // Get typing module and _GenericAlias let typing_module = vm.import("typing", 0)?; let generic_type = typing_module.get_attr("Generic", vm)?; - - // Call typing._GenericAlias(Generic, type_params) let generic_alias_class = typing_module.get_attr("_GenericAlias", vm)?; - let args = if let Ok(tuple) = type_params.try_to_ref::(vm) { + let params = if let Ok(tuple) = type_params.try_to_ref::(vm) { tuple.to_owned() } else { PyTuple::new_ref(vec![type_params], &vm.ctx) }; - // Create _GenericAlias instance + let args = crate::stdlib::typing::unpack_typevartuples(¶ms, vm)?; + generic_alias_class.call((generic_type, args.to_pyobject(vm)), vm) } diff --git a/crates/vm/src/stdlib/typevar.rs b/crates/vm/src/stdlib/typevar.rs index 33513b25027..1b3b5fecdd4 100644 --- a/crates/vm/src/stdlib/typevar.rs +++ b/crates/vm/src/stdlib/typevar.rs @@ -23,6 +23,25 @@ pub(crate) mod typevar { call_typing_func_object(vm, "_type_check", (arg, message_str)) } + fn variance_repr( + name: &str, + infer_variance: bool, + covariant: bool, + contravariant: bool, + ) -> String { + if infer_variance { + return name.to_string(); + } + let prefix = if covariant { + '+' + } else if contravariant { + '-' + } else { + '~' + }; + format!("{prefix}{name}") + } + /// Get the module of the caller frame, similar to CPython's caller() function. /// Returns the module name or None if not found. /// @@ -270,14 +289,12 @@ pub(crate) mod typevar { #[inline(always)] fn repr_str(zelf: &crate::Py, vm: &VirtualMachine) -> PyResult { let name = zelf.name.str(vm)?; - let repr = if zelf.covariant { - format!("+{name}") - } else if zelf.contravariant { - format!("-{name}") - } else { - format!("~{name}") - }; - Ok(repr) + Ok(variance_repr( + name.as_str(), + zelf.infer_variance, + zelf.covariant, + zelf.contravariant, + )) } } @@ -601,7 +618,10 @@ pub(crate) mod typevar { return Err(vm.new_type_error("ParamSpec() takes at most 1 positional argument")); }; - let bound = kwargs.swap_remove("bound"); + let bound = kwargs + .swap_remove("bound") + .map(|b| type_check(b, "Bound must be a type.", vm)) + .transpose()?; let covariant = kwargs .swap_remove("covariant") .map(|v| v.try_to_bool(vm)) @@ -665,7 +685,12 @@ pub(crate) mod typevar { #[inline(always)] fn repr_str(zelf: &crate::Py, vm: &VirtualMachine) -> PyResult { let name = zelf.__name__().str(vm)?; - Ok(format!("~{name}")) + Ok(variance_repr( + name.as_str(), + zelf.infer_variance, + zelf.covariant, + zelf.contravariant, + )) } } @@ -902,35 +927,19 @@ pub(crate) mod typevar { op: PyComparisonOp, vm: &VirtualMachine, ) -> PyResult { - fn eq( - zelf: &crate::Py, - other: PyObjectRef, - _vm: &VirtualMachine, - ) -> PyResult { - // First check if other is also ParamSpecArgs - if let Ok(other_args) = other.downcast::() { - // Check if they have the same origin - return Ok(zelf.__origin__.is(&other_args.__origin__)); - } - Ok(false) - } - match op { - PyComparisonOp::Eq => { - if let Ok(result) = eq(zelf, other.to_owned(), vm) { - Ok(result.into()) - } else { - Ok(PyComparisonValue::NotImplemented) - } - } - PyComparisonOp::Ne => { - if let Ok(result) = eq(zelf, other.to_owned(), vm) { - Ok((!result).into()) - } else { - Ok(PyComparisonValue::NotImplemented) - } + op.eq_only(|| { + if other.class().is(zelf.class()) + && let Some(other_args) = other.downcast_ref::() + { + let eq = zelf.__origin__.rich_compare_bool( + &other_args.__origin__, + PyComparisonOp::Eq, + vm, + )?; + return Ok(PyComparisonValue::Implemented(eq)); } - _ => Ok(PyComparisonValue::NotImplemented), - } + Ok(PyComparisonValue::NotImplemented) + }) } } @@ -981,35 +990,19 @@ pub(crate) mod typevar { op: PyComparisonOp, vm: &VirtualMachine, ) -> PyResult { - fn eq( - zelf: &crate::Py, - other: PyObjectRef, - _vm: &VirtualMachine, - ) -> PyResult { - // First check if other is also ParamSpecKwargs - if let Ok(other_kwargs) = other.downcast::() { - // Check if they have the same origin - return Ok(zelf.__origin__.is(&other_kwargs.__origin__)); - } - Ok(false) - } - match op { - PyComparisonOp::Eq => { - if let Ok(result) = eq(zelf, other.to_owned(), vm) { - Ok(result.into()) - } else { - Ok(PyComparisonValue::NotImplemented) - } - } - PyComparisonOp::Ne => { - if let Ok(result) = eq(zelf, other.to_owned(), vm) { - Ok((!result).into()) - } else { - Ok(PyComparisonValue::NotImplemented) - } + op.eq_only(|| { + if other.class().is(zelf.class()) + && let Some(other_kwargs) = other.downcast_ref::() + { + let eq = zelf.__origin__.rich_compare_bool( + &other_kwargs.__origin__, + PyComparisonOp::Eq, + vm, + )?; + return Ok(PyComparisonValue::Implemented(eq)); } - _ => Ok(PyComparisonValue::NotImplemented), - } + Ok(PyComparisonValue::NotImplemented) + }) } } diff --git a/crates/vm/src/stdlib/typing.rs b/crates/vm/src/stdlib/typing.rs index 6887106b4b5..bafc02e5764 100644 --- a/crates/vm/src/stdlib/typing.rs +++ b/crates/vm/src/stdlib/typing.rs @@ -56,7 +56,7 @@ pub(crate) mod decl { #[derive(Debug, PyPayload)] pub struct NoDefault; - #[pyclass(with(Constructor, Representable), flags(BASETYPE))] + #[pyclass(with(Constructor, Representable), flags(IMMUTABLETYPE))] impl NoDefault { #[pymethod] fn __reduce__(&self, _vm: &VirtualMachine) -> String { @@ -467,7 +467,10 @@ pub(crate) mod decl { } /// Wrap TypeVarTuples in Unpack[], matching unpack_typevartuples() - fn unpack_typevartuples(type_params: &PyTupleRef, vm: &VirtualMachine) -> PyResult { + pub(crate) fn unpack_typevartuples( + type_params: &PyTupleRef, + vm: &VirtualMachine, + ) -> PyResult { let has_tvt = type_params .iter() .any(|p| p.downcastable::()); diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index 53750610cae..ae00158aeb4 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -534,7 +534,15 @@ pub(crate) fn richcompare_wrapper( } fn iter_wrapper(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm.call_special_method(&zelf, identifier!(vm, __iter__), ()) + // slot_tp_iter: if __iter__ is None, the type is explicitly not iterable + let cls = zelf.class(); + let iter_attr = cls.get_attr(identifier!(vm, __iter__)); + match iter_attr { + Some(attr) if vm.is_none(&attr) => { + Err(vm.new_type_error(format!("'{}' object is not iterable", cls.name()))) + } + _ => vm.call_special_method(&zelf, identifier!(vm, __iter__), ()), + } } fn bool_wrapper(num: PyNumber<'_>, vm: &VirtualMachine) -> PyResult {