From 400696c0fdd63f578366bf8ed06ff756ce5430b9 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 3 Feb 2026 14:15:58 +0900 Subject: [PATCH 1/8] --exclude rustpython-venvlauncher --- AGENTS.md | 2 +- DEVELOPMENT.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 89326ef35ad..85e839a8538 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ rm -r target/debug/build/rustpython-* && find . | grep -E "\.pyc$" | xargs rm -r ```bash # Run Rust unit tests -cargo test --workspace --exclude rustpython_wasm +cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher # Run Python snippets tests (debug mode recommended for faster compilation) cargo run -- extra_tests/snippets/builtin_bytes.py diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 24c149eebef..7573f0f2640 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -65,7 +65,7 @@ $ pytest -v Rust unit tests can be run with `cargo`: ```shell -$ cargo test --workspace --exclude rustpython_wasm +$ cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher ``` Python unit tests can be run by compiling RustPython and running the test module: From 0da5931353795f8286176804b70740066ab55da3 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Sun, 1 Feb 2026 23:57:00 +0900 Subject: [PATCH 2/8] fix unparse --- crates/codegen/src/unparse.rs | 17 +++++++++++++++- crates/vm/src/stdlib/ast/expression.rs | 21 +++++++++++++++++++- crates/vm/src/stdlib/ast/other.rs | 2 +- crates/vm/src/stdlib/ast/python.rs | 27 ++++++++++++++++++++++++++ crates/vm/src/stdlib/builtins.rs | 7 +++++++ 5 files changed, 71 insertions(+), 3 deletions(-) diff --git a/crates/codegen/src/unparse.rs b/crates/codegen/src/unparse.rs index eef2128587a..cb9f3783fc3 100644 --- a/crates/codegen/src/unparse.rs +++ b/crates/codegen/src/unparse.rs @@ -363,7 +363,9 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.p(")")?; } ast::Expr::FString(ast::ExprFString { value, .. }) => self.unparse_fstring(value)?, - ast::Expr::TString(_) => self.p("t\"\"")?, + ast::Expr::TString(ast::ExprTString { value, .. }) => { + self.unparse_tstring(value)? + } ast::Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) => { if value.is_unicode() { self.p("u")? @@ -626,6 +628,19 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { .str_repr() .write(self.f) } + + fn unparse_tstring(&mut self, value: &ast::TStringValue) -> fmt::Result { + self.p("t")?; + let body = fmt::from_fn(|f| { + value.iter().try_for_each(|tstring| { + Unparser::new(f, self.source).unparse_fstring_body(&tstring.elements) + }) + }) + .to_string(); + UnicodeEscape::new_repr(body.as_str().as_ref()) + .str_repr() + .write(self.f) + } } pub struct UnparseExpr<'a> { diff --git a/crates/vm/src/stdlib/ast/expression.rs b/crates/vm/src/stdlib/ast/expression.rs index 3bf1470795d..fc1831bf597 100644 --- a/crates/vm/src/stdlib/ast/expression.rs +++ b/crates/vm/src/stdlib/ast/expression.rs @@ -327,7 +327,26 @@ impl Node for ast::ExprLambda { .into_ref_with_type(vm, pyast::NodeExprLambda::static_type().to_owned()) .unwrap(); let dict = node.as_object().dict().unwrap(); - dict.set_item("args", parameters.ast_to_object(vm, source_file), vm) + // Lambda with no parameters should have an empty arguments object, not None + let args = match parameters { + Some(params) => params.ast_to_object(vm, source_file), + None => { + // Create an empty arguments object + let args_node = NodeAst + .into_ref_with_type(vm, pyast::NodeArguments::static_type().to_owned()) + .unwrap(); + let args_dict = args_node.as_object().dict().unwrap(); + args_dict.set_item("posonlyargs", vm.ctx.new_list(vec![]).into(), vm).unwrap(); + args_dict.set_item("args", vm.ctx.new_list(vec![]).into(), vm).unwrap(); + args_dict.set_item("vararg", vm.ctx.none(), vm).unwrap(); + args_dict.set_item("kwonlyargs", vm.ctx.new_list(vec![]).into(), vm).unwrap(); + args_dict.set_item("kw_defaults", vm.ctx.new_list(vec![]).into(), vm).unwrap(); + args_dict.set_item("kwarg", vm.ctx.none(), vm).unwrap(); + args_dict.set_item("defaults", vm.ctx.new_list(vec![]).into(), vm).unwrap(); + args_node.into() + } + }; + dict.set_item("args", args, vm) .unwrap(); dict.set_item("body", body.ast_to_object(vm, source_file), vm) .unwrap(); diff --git a/crates/vm/src/stdlib/ast/other.rs b/crates/vm/src/stdlib/ast/other.rs index 8a89a740682..c7a1974351a 100644 --- a/crates/vm/src/stdlib/ast/other.rs +++ b/crates/vm/src/stdlib/ast/other.rs @@ -3,7 +3,7 @@ use rustpython_compiler_core::SourceFile; impl Node for ast::ConversionFlag { fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { - vm.ctx.new_int(self as u8).into() + vm.ctx.new_int(self as i8).into() } fn ast_from_object( diff --git a/crates/vm/src/stdlib/ast/python.rs b/crates/vm/src/stdlib/ast/python.rs index 6c38b00f9ad..152ae9abcad 100644 --- a/crates/vm/src/stdlib/ast/python.rs +++ b/crates/vm/src/stdlib/ast/python.rs @@ -65,8 +65,13 @@ pub(crate) mod _ast { if fields.len() == 1 { "" } else { "s" }, ))); } + + // Track which fields were set + let mut set_fields = std::collections::HashSet::new(); + for (name, arg) in fields.iter().zip(args.args) { zelf.set_attr(name, arg, vm)?; + set_fields.insert(name.as_str().to_string()); } for (key, value) in args.kwargs { if let Some(pos) = fields.iter().position(|f| f.as_str() == key) @@ -78,9 +83,31 @@ pub(crate) mod _ast { key ))); } + set_fields.insert(key.clone()); zelf.set_attr(vm.ctx.intern_str(key), value, vm)?; } + // Set default values for fields that weren't provided + let class_name = &*zelf.class().name(); + if class_name == "Module" && !set_fields.contains("type_ignores") { + zelf.set_attr("type_ignores", vm.ctx.new_list(vec![]), vm)?; + } + if class_name == "ImportFrom" && !set_fields.contains("level") { + zelf.set_attr("level", vm.ctx.new_int(0), vm)?; + } + if class_name == "alias" && !set_fields.contains("asname") { + zelf.set_attr("asname", vm.ctx.none(), vm)?; + } + // Set type_comment to None for nodes that support it + if !set_fields.contains("type_comment") { + match class_name { + "FunctionDef" | "AsyncFunctionDef" | "For" | "AsyncFor" | "With" | "AsyncWith" | "arg" => { + zelf.set_attr("type_comment", vm.ctx.none(), vm)?; + } + _ => {} + } + } + Ok(()) } diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index 1f14f6f5b04..6c72c4c691a 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -141,6 +141,13 @@ mod builtins { .source .fast_isinstance(&ast::NodeAst::make_class(&vm.ctx)) { + // If PyCF_ONLY_AST is set, just return the AST node as-is + use num_traits::Zero; + let flags = args.flags.map_or(Ok(0), |v| v.try_to_primitive(vm))?; + if !(flags & ast::PY_COMPILE_FLAG_AST_ONLY).is_zero() { + return Ok(args.source); + } + #[cfg(not(feature = "rustpython-codegen"))] { return Err(vm.new_type_error(CODEGEN_NOT_SUPPORTED.to_owned())); From 1876ac88e04d46d819e2da322f44e92b842832e1 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 2 Feb 2026 14:59:39 +0900 Subject: [PATCH 3/8] Fix compiler panics --- crates/codegen/src/compile.rs | 20 +++++++++++--------- crates/codegen/src/string_parser.rs | 11 +++++++++-- crates/vm/src/protocol/object.rs | 4 ++++ 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 02167667a8b..7e2b25ccbef 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -5039,8 +5039,9 @@ impl Compiler { } fn compile_error_forbidden_name(&mut self, name: &str) -> CodegenError { - // TODO: make into error (fine for now since it realistically errors out earlier) - panic!("Failing due to forbidden name {name:?}"); + self.error(CodegenErrorType::SyntaxError(format!( + "cannot use forbidden name '{name}' in pattern" + ))) } /// Ensures that `pc.fail_pop` has at least `n + 1` entries. @@ -5387,12 +5388,9 @@ impl Compiler { // Check for too many sub-patterns. if nargs > u32::MAX as usize || (nargs + n_attrs).saturating_sub(1) > i32::MAX as usize { - let msg = format!( - "too many sub-patterns in class pattern {:?}", - match_class.cls - ); - panic!("{}", msg); - // return self.compiler_error(&msg); + return Err(self.error(CodegenErrorType::SyntaxError( + "too many sub-patterns in class pattern".to_owned(), + ))); } // Validate keyword attributes if any. @@ -5677,7 +5675,11 @@ impl Compiler { // Ensure the pattern is a MatchOr. let end = self.new_block(); // Create a new jump target label. let size = p.patterns.len(); - assert!(size > 1, "MatchOr must have more than one alternative"); + if size <= 1 { + return Err(self.error(CodegenErrorType::SyntaxError( + "MatchOr requires at least 2 patterns".to_owned(), + ))); + } // Save the current pattern context. let old_pc = pc.clone(); diff --git a/crates/codegen/src/string_parser.rs b/crates/codegen/src/string_parser.rs index 7e1558d2b17..a7ad8c35a46 100644 --- a/crates/codegen/src/string_parser.rs +++ b/crates/codegen/src/string_parser.rs @@ -273,8 +273,15 @@ impl StringParser { } pub(crate) fn parse_string_literal(source: &str, flags: ast::AnyStringFlags) -> Box { - let source = &source[flags.opener_len().to_usize()..]; - let source = &source[..source.len() - flags.quote_len().to_usize()]; + let opener_len = flags.opener_len().to_usize(); + let quote_len = flags.quote_len().to_usize(); + if source.len() < opener_len + quote_len { + // Source unavailable (e.g., compiling from an AST object with no + // backing source text). Return the raw source as-is. + return Box::::from(source); + } + let source = &source[opener_len..]; + let source = &source[..source.len() - quote_len]; StringParser::new(source.into(), flags) .parse_string() .unwrap_or_else(|x| match x {}) diff --git a/crates/vm/src/protocol/object.rs b/crates/vm/src/protocol/object.rs index ec1a6f55969..02a712979f2 100644 --- a/crates/vm/src/protocol/object.rs +++ b/crates/vm/src/protocol/object.rs @@ -707,6 +707,10 @@ impl PyObject { { return class_getitem.call((needle,), vm); } + return Err(vm.new_type_error(format!( + "type '{}' is not subscriptable", + self.downcast_ref::().unwrap().name() + ))); } Err(vm.new_type_error(format!("'{}' object is not subscriptable", self.class()))) } From 63bbb3c80455c7dfdd4c48b613f14571ae086c2f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 2 Feb 2026 14:59:50 +0900 Subject: [PATCH 4/8] fix ast fields including type_comments --- crates/vm/src/stdlib/ast/parameter.rs | 5 +- crates/vm/src/stdlib/ast/pyast.rs | 12 +++- crates/vm/src/stdlib/ast/python.rs | 66 +++++++++++++++------ crates/vm/src/stdlib/ast/statement.rs | 16 ++--- crates/vm/src/stdlib/ast/type_parameters.rs | 8 ++- crates/vm/src/stdlib/builtins.rs | 19 +++++- 6 files changed, 95 insertions(+), 31 deletions(-) diff --git a/crates/vm/src/stdlib/ast/parameter.rs b/crates/vm/src/stdlib/ast/parameter.rs index 1e411d41ab6..b1942d833ba 100644 --- a/crates/vm/src/stdlib/ast/parameter.rs +++ b/crates/vm/src/stdlib/ast/parameter.rs @@ -126,8 +126,9 @@ impl Node for ast::Parameter { _vm, ) .unwrap(); - // dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - // .unwrap(); + // Ruff AST doesn't track type_comment, so always set to None + dict.set_item("type_comment", _vm.ctx.none(), _vm) + .unwrap(); node_add_location(&dict, range, _vm, source_file); node.into() } diff --git a/crates/vm/src/stdlib/ast/pyast.rs b/crates/vm/src/stdlib/ast/pyast.rs index 2131df29b96..d6f995f6f72 100644 --- a/crates/vm/src/stdlib/ast/pyast.rs +++ b/crates/vm/src/stdlib/ast/pyast.rs @@ -37,6 +37,12 @@ macro_rules! impl_node { ),* ]).into(), ); + + // Signal that this is a built-in AST node with field defaults + class.set_attr( + ctx.intern_str("_field_types"), + ctx.new_dict().into(), + ); } } }; @@ -902,21 +908,21 @@ impl_node!( impl_node!( #[pyclass(module = "_ast", name = "TypeVar", base = NodeTypeParam)] pub(crate) struct NodeTypeParamTypeVar, - fields: ["name", "bound"], + fields: ["name", "bound", "default_value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( #[pyclass(module = "_ast", name = "ParamSpec", base = NodeTypeParam)] pub(crate) struct NodeTypeParamParamSpec, - fields: ["name"], + fields: ["name", "default_value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( #[pyclass(module = "_ast", name = "TypeVarTuple", base = NodeTypeParam)] pub(crate) struct NodeTypeParamTypeVarTuple, - fields: ["name"], + fields: ["name", "default_value"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); diff --git a/crates/vm/src/stdlib/ast/python.rs b/crates/vm/src/stdlib/ast/python.rs index 152ae9abcad..924026735d7 100644 --- a/crates/vm/src/stdlib/ast/python.rs +++ b/crates/vm/src/stdlib/ast/python.rs @@ -87,24 +87,56 @@ pub(crate) mod _ast { zelf.set_attr(vm.ctx.intern_str(key), value, vm)?; } - // Set default values for fields that weren't provided - let class_name = &*zelf.class().name(); - if class_name == "Module" && !set_fields.contains("type_ignores") { - zelf.set_attr("type_ignores", vm.ctx.new_list(vec![]), vm)?; - } - if class_name == "ImportFrom" && !set_fields.contains("level") { - zelf.set_attr("level", vm.ctx.new_int(0), vm)?; - } - if class_name == "alias" && !set_fields.contains("asname") { - zelf.set_attr("asname", vm.ctx.none(), vm)?; - } - // Set type_comment to None for nodes that support it - if !set_fields.contains("type_comment") { - match class_name { - "FunctionDef" | "AsyncFunctionDef" | "For" | "AsyncFor" | "With" | "AsyncWith" | "arg" => { - zelf.set_attr("type_comment", vm.ctx.none(), vm)?; + // Set default values only for built-in AST nodes (_field_types present). + // Custom AST subclasses without _field_types do NOT get automatic defaults. + let has_field_types = zelf.class().get_attr(vm.ctx.intern_str("_field_types")).is_some(); + if has_field_types { + // ASDL list fields (type*) default to empty list, + // optional fields (type?) default to None. + const LIST_FIELDS: &[&str] = &[ + "args", + "argtypes", + "bases", + "body", + "cases", + "comparators", + "decorator_list", + "defaults", + "elts", + "finalbody", + "generators", + "handlers", + "ifs", + "items", + "keys", + "kw_defaults", + "keywords", + "kwonlyargs", + "names", + "orelse", + "ops", + "posonlyargs", + "targets", + "type_ignores", + "type_params", + "values", + ]; + + for field in &fields { + if !set_fields.contains(field.as_str()) { + let default: PyObjectRef = if LIST_FIELDS.contains(&field.as_str()) { + vm.ctx.new_list(vec![]).into() + } else { + vm.ctx.none() + }; + zelf.set_attr(vm.ctx.intern_str(field.as_str()), default, vm)?; } - _ => {} + } + + // Special defaults that are not None or empty list + let class_name = &*zelf.class().name(); + if class_name == "ImportFrom" && !set_fields.contains("level") { + zelf.set_attr("level", vm.ctx.new_int(0), vm)?; } } diff --git a/crates/vm/src/stdlib/ast/statement.rs b/crates/vm/src/stdlib/ast/statement.rs index b7bc692dd2e..620a8317878 100644 --- a/crates/vm/src/stdlib/ast/statement.rs +++ b/crates/vm/src/stdlib/ast/statement.rs @@ -182,9 +182,9 @@ impl Node for ast::StmtFunctionDef { .unwrap(); dict.set_item("returns", returns.ast_to_object(vm, source_file), vm) .unwrap(); - // TODO: Ruff ignores type_comment during parsing - // dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - // .unwrap(); + // Ruff AST doesn't track type_comment, so always set to None + dict.set_item("type_comment", vm.ctx.none(), vm) + .unwrap(); dict.set_item( "type_params", type_params @@ -647,8 +647,9 @@ impl Node for ast::StmtFor { .unwrap(); dict.set_item("orelse", orelse.ast_to_object(_vm, source_file), _vm) .unwrap(); - // dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - // .unwrap(); + // Ruff AST doesn't track type_comment, so always set to None + dict.set_item("type_comment", _vm.ctx.none(), _vm) + .unwrap(); node_add_location(&dict, _range, _vm, source_file); node.into() } @@ -799,8 +800,9 @@ impl Node for ast::StmtWith { .unwrap(); dict.set_item("body", body.ast_to_object(_vm, source_file), _vm) .unwrap(); - // dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - // .unwrap(); + // Ruff AST doesn't track type_comment, so always set to None + dict.set_item("type_comment", _vm.ctx.none(), _vm) + .unwrap(); node_add_location(&dict, _range, _vm, source_file); node.into() } diff --git a/crates/vm/src/stdlib/ast/type_parameters.rs b/crates/vm/src/stdlib/ast/type_parameters.rs index 4801a9a4b28..ccfbf464909 100644 --- a/crates/vm/src/stdlib/ast/type_parameters.rs +++ b/crates/vm/src/stdlib/ast/type_parameters.rs @@ -78,7 +78,7 @@ impl Node for ast::TypeParamTypeVar { name, bound, range: _range, - default: _, + default, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeTypeParamTypeVar::static_type().to_owned()) @@ -88,6 +88,12 @@ impl Node for ast::TypeParamTypeVar { .unwrap(); dict.set_item("bound", bound.ast_to_object(_vm, source_file), _vm) .unwrap(); + dict.set_item( + "default_value", + default.ast_to_object(_vm, source_file), + _vm, + ) + .unwrap(); node_add_location(&dict, _range, _vm, source_file); node.into() } diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index 6c72c4c691a..8bfbffcc613 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -141,10 +141,27 @@ mod builtins { .source .fast_isinstance(&ast::NodeAst::make_class(&vm.ctx)) { - // If PyCF_ONLY_AST is set, just return the AST node as-is use num_traits::Zero; let flags = args.flags.map_or(Ok(0), |v| v.try_to_primitive(vm))?; + // compile(ast_node, ..., PyCF_ONLY_AST) returns the AST after validation if !(flags & ast::PY_COMPILE_FLAG_AST_ONLY).is_zero() { + let expected_type = match mode_str { + "exec" => "Module", + "eval" => "Expression", + "single" => "Interactive", + "func_type" => "FunctionType", + _ => { + return Err(vm.new_value_error(format!( + "compile() mode must be 'exec', 'eval', 'single' or 'func_type', got '{mode_str}'" + ))); + } + }; + let cls_name = args.source.class().name().to_string(); + if cls_name != expected_type { + return Err(vm.new_type_error(format!( + "expected {expected_type} node, got {cls_name}" + ))); + } return Ok(args.source); } From 69c19f7cd19d32faf1906eb5a1b03fbc64242b0d Mon Sep 17 00:00:00 2001 From: CPython Developers <> Date: Mon, 2 Feb 2026 00:03:28 +0900 Subject: [PATCH 5/8] Update _ast_unparse from v3.14.2 --- Lib/test/test_unparse.py | 1072 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 1072 insertions(+) create mode 100644 Lib/test/test_unparse.py diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py new file mode 100644 index 00000000000..c7480fb3476 --- /dev/null +++ b/Lib/test/test_unparse.py @@ -0,0 +1,1072 @@ +"""Tests for ast.unparse.""" + +import unittest +import test.support +import pathlib +import random +import tokenize +import warnings +import ast +from test.support.ast_helper import ASTTestMixin + + +def read_pyfile(filename): + """Read and return the contents of a Python source file (as a + string), taking into account the file encoding.""" + with tokenize.open(filename) as stream: + return stream.read() + + +for_else = """\ +def f(): + for x in range(10): + break + else: + y = 2 + z = 3 +""" + +while_else = """\ +def g(): + while True: + break + else: + y = 2 + z = 3 +""" + +relative_import = """\ +from . import fred +from .. import barney +from .australia import shrimp as prawns +""" + +nonlocal_ex = """\ +def f(): + x = 1 + def g(): + nonlocal x + x = 2 + y = 7 + def h(): + nonlocal x, y +""" + +# also acts as test for 'except ... as ...' +raise_from = """\ +try: + 1 / 0 +except ZeroDivisionError as e: + raise ArithmeticError from e +""" + +class_decorator = """\ +@f1(arg) +@f2 +class Foo: pass +""" + +elif1 = """\ +if cond1: + suite1 +elif cond2: + suite2 +else: + suite3 +""" + +elif2 = """\ +if cond1: + suite1 +elif cond2: + suite2 +""" + +try_except_finally = """\ +try: + suite1 +except ex1: + suite2 +except ex2: + suite3 +else: + suite4 +finally: + suite5 +""" + +try_except_star_finally = """\ +try: + suite1 +except* ex1: + suite2 +except* ex2: + suite3 +else: + suite4 +finally: + suite5 +""" + +with_simple = """\ +with f(): + suite1 +""" + +with_as = """\ +with f() as x: + suite1 +""" + +with_two_items = """\ +with f() as x, g() as y: + suite1 +""" + +docstring_prefixes = ( + "", + "class foo:\n ", + "def foo():\n ", + "async def foo():\n ", +) + +class ASTTestCase(ASTTestMixin, unittest.TestCase): + def check_ast_roundtrip(self, code1, **kwargs): + with self.subTest(code1=code1, ast_parse_kwargs=kwargs): + ast1 = ast.parse(code1, **kwargs) + code2 = ast.unparse(ast1) + ast2 = ast.parse(code2, **kwargs) + self.assertASTEqual(ast1, ast2) + + def check_invalid(self, node, raises=ValueError): + with self.subTest(node=node): + self.assertRaises(raises, ast.unparse, node) + + def get_source(self, code1, code2=None, **kwargs): + code2 = code2 or code1 + code1 = ast.unparse(ast.parse(code1, **kwargs)) + return code1, code2 + + def check_src_roundtrip(self, code1, code2=None, **kwargs): + code1, code2 = self.get_source(code1, code2, **kwargs) + with self.subTest(code1=code1, code2=code2): + self.assertEqual(code2, code1) + + def check_src_dont_roundtrip(self, code1, code2=None): + code1, code2 = self.get_source(code1, code2) + with self.subTest(code1=code1, code2=code2): + self.assertNotEqual(code2, code1) + +class UnparseTestCase(ASTTestCase): + # Tests for specific bugs found in earlier versions of unparse + + def test_fstrings(self): + self.check_ast_roundtrip("f'a'") + self.check_ast_roundtrip("f'{{}}'") + self.check_ast_roundtrip("f'{{5}}'") + self.check_ast_roundtrip("f'{{5}}5'") + self.check_ast_roundtrip("f'X{{}}X'") + self.check_ast_roundtrip("f'{a}'") + self.check_ast_roundtrip("f'{ {1:2}}'") + self.check_ast_roundtrip("f'a{a}a'") + self.check_ast_roundtrip("f'a{a}{a}a'") + self.check_ast_roundtrip("f'a{a}a{a}a'") + self.check_ast_roundtrip("f'{a!r}x{a!s}12{{}}{a!a}'") + self.check_ast_roundtrip("f'{a:10}'") + self.check_ast_roundtrip("f'{a:100_000{10}}'") + self.check_ast_roundtrip("f'{a!r:10}'") + self.check_ast_roundtrip("f'{a:a{b}10}'") + self.check_ast_roundtrip( + "f'a{b}{c!s}{d!r}{e!a}{f:a}{g:a{b}}{h!s:a}" + "{j!s:{a}b}{k!s:a{b}c}{l!a:{b}c{d}}{x+y=}'" + ) + + def test_fstrings_special_chars(self): + # See issue 25180 + self.check_ast_roundtrip(r"""f'{f"{0}"*3}'""") + self.check_ast_roundtrip(r"""f'{f"{y}"*3}'""") + self.check_ast_roundtrip("""f''""") + self.check_ast_roundtrip('''f"""'end' "quote\\""""''') + + def test_fstrings_complicated(self): + # See issue 28002 + self.check_ast_roundtrip("""f'''{"'"}'''""") + self.check_ast_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\'''') + self.check_ast_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-'single quote\\'\'\'\'''') + self.check_ast_roundtrip('f"""{\'\'\'\n\'\'\'}"""') + self.check_ast_roundtrip('f"""{g(\'\'\'\n\'\'\')}"""') + self.check_ast_roundtrip('''f"a\\r\\nb"''') + self.check_ast_roundtrip('''f"\\u2028{'x'}"''') + + def test_fstrings_pep701(self): + self.check_ast_roundtrip('f" something { my_dict["key"] } something else "') + self.check_ast_roundtrip('f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_tstrings(self): + self.check_ast_roundtrip("t'foo'") + self.check_ast_roundtrip("t'foo {bar}'") + self.check_ast_roundtrip("t'foo {bar!s:.2f}'") + self.check_ast_roundtrip("t'{a + b}'") + self.check_ast_roundtrip("t'{a + b:x}'") + self.check_ast_roundtrip("t'{a + b!s}'") + self.check_ast_roundtrip("t'{ {a}}'") + self.check_ast_roundtrip("t'{ {a}=}'") + self.check_ast_roundtrip("t'{{a}}'") + self.check_ast_roundtrip("t''") + self.check_ast_roundtrip('t""') + self.check_ast_roundtrip("t'{(lambda x: x)}'") + self.check_ast_roundtrip("t'{t'{x}'}'") + + def test_tstring_with_nonsensical_str_field(self): + # `value` suggests that the original code is `t'{test1}`, but `str` suggests otherwise + self.assertEqual( + ast.unparse( + ast.TemplateStr( + values=[ + ast.Interpolation( + value=ast.Name(id="test1", ctx=ast.Load()), str="test2", conversion=-1 + ) + ] + ) + ), + "t'{test2}'", + ) + + def test_tstring_with_none_str_field(self): + self.assertEqual( + ast.unparse( + ast.TemplateStr( + [ast.Interpolation(value=ast.Name(id="test1"), str=None, conversion=-1)] + ) + ), + "t'{test1}'", + ) + self.assertEqual( + ast.unparse( + ast.TemplateStr( + [ + ast.Interpolation( + value=ast.Lambda( + args=ast.arguments(args=[ast.arg(arg="x")]), + body=ast.Name(id="x"), + ), + str=None, + conversion=-1, + ) + ] + ) + ), + "t'{(lambda x: x)}'", + ) + self.assertEqual( + ast.unparse( + ast.TemplateStr( + values=[ + ast.Interpolation( + value=ast.TemplateStr( + # `str` field kept here + [ast.Interpolation(value=ast.Name(id="x"), str="y", conversion=-1)] + ), + str=None, + conversion=-1, + ) + ] + ) + ), + '''t"{t'{y}'}"''', + ) + self.assertEqual( + ast.unparse( + ast.TemplateStr( + values=[ + ast.Interpolation( + value=ast.TemplateStr( + [ast.Interpolation(value=ast.Name(id="x"), str=None, conversion=-1)] + ), + str=None, + conversion=-1, + ) + ] + ) + ), + '''t"{t'{x}'}"''', + ) + self.assertEqual( + ast.unparse(ast.TemplateStr( + [ast.Interpolation(value=ast.Constant(value="foo"), str=None, conversion=114)] + )), + '''t"{'foo'!r}"''', + ) + + def test_strings(self): + self.check_ast_roundtrip("u'foo'") + self.check_ast_roundtrip("r'foo'") + self.check_ast_roundtrip("b'foo'") + + def test_del_statement(self): + self.check_ast_roundtrip("del x, y, z") + + def test_shifts(self): + self.check_ast_roundtrip("45 << 2") + self.check_ast_roundtrip("13 >> 7") + + def test_for_else(self): + self.check_ast_roundtrip(for_else) + + def test_while_else(self): + self.check_ast_roundtrip(while_else) + + def test_unary_parens(self): + self.check_ast_roundtrip("(-1)**7") + self.check_ast_roundtrip("(-1.)**8") + self.check_ast_roundtrip("(-1j)**6") + self.check_ast_roundtrip("not True or False") + self.check_ast_roundtrip("True or not False") + + def test_integer_parens(self): + self.check_ast_roundtrip("3 .__abs__()") + + def test_huge_float(self): + self.check_ast_roundtrip("1e1000") + self.check_ast_roundtrip("-1e1000") + self.check_ast_roundtrip("1e1000j") + self.check_ast_roundtrip("-1e1000j") + + def test_nan(self): + self.assertASTEqual( + ast.parse(ast.unparse(ast.Constant(value=float('nan')))), + ast.parse('1e1000 - 1e1000') + ) + + def test_min_int(self): + self.check_ast_roundtrip(str(-(2 ** 31))) + self.check_ast_roundtrip(str(-(2 ** 63))) + + def test_imaginary_literals(self): + self.check_ast_roundtrip("7j") + self.check_ast_roundtrip("-7j") + self.check_ast_roundtrip("0j") + self.check_ast_roundtrip("-0j") + + def test_lambda_parentheses(self): + self.check_ast_roundtrip("(lambda: int)()") + + def test_chained_comparisons(self): + self.check_ast_roundtrip("1 < 4 <= 5") + self.check_ast_roundtrip("a is b is c is not d") + + def test_function_arguments(self): + self.check_ast_roundtrip("def f(): pass") + self.check_ast_roundtrip("def f(a): pass") + self.check_ast_roundtrip("def f(b = 2): pass") + self.check_ast_roundtrip("def f(a, b): pass") + self.check_ast_roundtrip("def f(a, b = 2): pass") + self.check_ast_roundtrip("def f(a = 5, b = 2): pass") + self.check_ast_roundtrip("def f(*, a = 1, b = 2): pass") + self.check_ast_roundtrip("def f(*, a = 1, b): pass") + self.check_ast_roundtrip("def f(*, a, b = 2): pass") + self.check_ast_roundtrip("def f(a, b = None, *, c, **kwds): pass") + self.check_ast_roundtrip("def f(a=2, *args, c=5, d, **kwds): pass") + self.check_ast_roundtrip("def f(*args, **kwargs): pass") + + def test_relative_import(self): + self.check_ast_roundtrip(relative_import) + + def test_nonlocal(self): + self.check_ast_roundtrip(nonlocal_ex) + + def test_raise_from(self): + self.check_ast_roundtrip(raise_from) + + def test_bytes(self): + self.check_ast_roundtrip("b'123'") + + def test_annotations(self): + self.check_ast_roundtrip("def f(a : int): pass") + self.check_ast_roundtrip("def f(a: int = 5): pass") + self.check_ast_roundtrip("def f(*args: [int]): pass") + self.check_ast_roundtrip("def f(**kwargs: dict): pass") + self.check_ast_roundtrip("def f() -> None: pass") + + def test_set_literal(self): + self.check_ast_roundtrip("{'a', 'b', 'c'}") + + def test_empty_set(self): + self.assertASTEqual( + ast.parse(ast.unparse(ast.Set(elts=[]))), + ast.parse('{*()}') + ) + + def test_set_comprehension(self): + self.check_ast_roundtrip("{x for x in range(5)}") + + def test_dict_comprehension(self): + self.check_ast_roundtrip("{x: x*x for x in range(10)}") + + def test_class_decorators(self): + self.check_ast_roundtrip(class_decorator) + + def test_class_definition(self): + self.check_ast_roundtrip("class A(metaclass=type, *[], **{}): pass") + + def test_elifs(self): + self.check_ast_roundtrip(elif1) + self.check_ast_roundtrip(elif2) + + def test_try_except_finally(self): + self.check_ast_roundtrip(try_except_finally) + + def test_try_except_star_finally(self): + self.check_ast_roundtrip(try_except_star_finally) + + def test_starred_assignment(self): + self.check_ast_roundtrip("a, *b, c = seq") + self.check_ast_roundtrip("a, (*b, c) = seq") + self.check_ast_roundtrip("a, *b[0], c = seq") + self.check_ast_roundtrip("a, *(b, c) = seq") + + def test_with_simple(self): + self.check_ast_roundtrip(with_simple) + + def test_with_as(self): + self.check_ast_roundtrip(with_as) + + def test_with_two_items(self): + self.check_ast_roundtrip(with_two_items) + + def test_dict_unpacking_in_dict(self): + # See issue 26489 + self.check_ast_roundtrip(r"""{**{'y': 2}, 'x': 1}""") + self.check_ast_roundtrip(r"""{**{'y': 2}, **{'x': 1}}""") + + def test_slices(self): + self.check_ast_roundtrip("a[i]") + self.check_ast_roundtrip("a[i,]") + self.check_ast_roundtrip("a[i, j]") + # The AST for these next two both look like `a[(*a,)]` + self.check_ast_roundtrip("a[(*a,)]") + self.check_ast_roundtrip("a[*a]") + self.check_ast_roundtrip("a[b, *a]") + self.check_ast_roundtrip("a[*a, c]") + self.check_ast_roundtrip("a[b, *a, c]") + self.check_ast_roundtrip("a[*a, *a]") + self.check_ast_roundtrip("a[b, *a, *a]") + self.check_ast_roundtrip("a[*a, b, *a]") + self.check_ast_roundtrip("a[*a, *a, b]") + self.check_ast_roundtrip("a[b, *a, *a, c]") + self.check_ast_roundtrip("a[(a:=b)]") + self.check_ast_roundtrip("a[(a:=b,c)]") + self.check_ast_roundtrip("a[()]") + self.check_ast_roundtrip("a[i:j]") + self.check_ast_roundtrip("a[:j]") + self.check_ast_roundtrip("a[i:]") + self.check_ast_roundtrip("a[i:j:k]") + self.check_ast_roundtrip("a[:j:k]") + self.check_ast_roundtrip("a[i::k]") + self.check_ast_roundtrip("a[i:j,]") + self.check_ast_roundtrip("a[i:j, k]") + + def test_invalid_raise(self): + self.check_invalid(ast.Raise(exc=None, cause=ast.Name(id="X", ctx=ast.Load()))) + + def test_invalid_fstring_value(self): + self.check_invalid( + ast.JoinedStr( + values=[ + ast.Name(id="test", ctx=ast.Load()), + ast.Constant(value="test") + ] + ) + ) + + def test_fstring_backslash(self): + # valid since Python 3.12 + self.assertEqual(ast.unparse( + ast.FormattedValue( + value=ast.Constant(value="\\\\"), + conversion=-1, + format_spec=None, + ) + ), "{'\\\\\\\\'}") + + def test_invalid_yield_from(self): + self.check_invalid(ast.YieldFrom(value=None)) + + def test_import_from_level_none(self): + tree = ast.ImportFrom(module='mod', names=[ast.alias(name='x')]) + self.assertEqual(ast.unparse(tree), "from mod import x") + tree = ast.ImportFrom(module='mod', names=[ast.alias(name='x')], level=None) + self.assertEqual(ast.unparse(tree), "from mod import x") + + def test_docstrings(self): + docstrings = ( + 'this ends with double quote"', + 'this includes a """triple quote"""', + '\r', + '\\r', + '\t', + '\\t', + '\n', + '\\n', + '\r\\r\t\\t\n\\n', + '""">>> content = \"\"\"blabla\"\"\" <<<"""', + r'foo\n\x00', + "' \\'\\'\\'\"\"\" \"\"\\'\\' \\'", + '🐍⛎𩸽üéş^\\\\X\\\\BB\N{LONG RIGHTWARDS SQUIGGLE ARROW}' + ) + for docstring in docstrings: + # check as Module docstrings for easy testing + self.check_ast_roundtrip(f"'''{docstring}'''") + + def test_constant_tuples(self): + locs = ast.fix_missing_locations + self.check_src_roundtrip( + locs(ast.Module([ast.Expr(ast.Constant(value=(1,)))])), "(1,)") + self.check_src_roundtrip( + locs(ast.Module([ast.Expr(ast.Constant(value=(1, 2, 3)))])), "(1, 2, 3)" + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_function_type(self): + for function_type in ( + "() -> int", + "(int, int) -> int", + "(Callable[complex], More[Complex(call.to_typevar())]) -> None" + ): + self.check_ast_roundtrip(function_type, mode="func_type") + + def test_type_comments(self): + for statement in ( + "a = 5 # type:", + "a = 5 # type: int", + "a = 5 # type: int and more", + "def x(): # type: () -> None\n\tpass", + "def x(y): # type: (int) -> None and more\n\tpass", + "async def x(): # type: () -> None\n\tpass", + "async def x(y): # type: (int) -> None and more\n\tpass", + "for x in y: # type: int\n\tpass", + "async for x in y: # type: int\n\tpass", + "with x(): # type: int\n\tpass", + "async with x(): # type: int\n\tpass" + ): + self.check_ast_roundtrip(statement, type_comments=True) + + def test_type_ignore(self): + for statement in ( + "a = 5 # type: ignore", + "a = 5 # type: ignore and more", + "def x(): # type: ignore\n\tpass", + "def x(y): # type: ignore and more\n\tpass", + "async def x(): # type: ignore\n\tpass", + "async def x(y): # type: ignore and more\n\tpass", + "for x in y: # type: ignore\n\tpass", + "async for x in y: # type: ignore\n\tpass", + "with x(): # type: ignore\n\tpass", + "async with x(): # type: ignore\n\tpass" + ): + self.check_ast_roundtrip(statement, type_comments=True) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'TypeVar' object has no attribute 'default_value' + def test_unparse_interactive_semicolons(self): + # gh-129598: Fix ast.unparse() when ast.Interactive contains multiple statements + self.check_src_roundtrip("i = 1; 'expr'; raise Exception", mode='single') + self.check_src_roundtrip("i: int = 1; j: float = 0; k += l", mode='single') + combinable = ( + "'expr'", + "(i := 1)", + "import foo", + "from foo import bar", + "i = 1", + "i += 1", + "i: int = 1", + "return i", + "pass", + "break", + "continue", + "del i", + "assert i", + "global i", + "nonlocal j", + "await i", + "yield i", + "yield from i", + "raise i", + "type t[T] = ...", + "i", + ) + for a in combinable: + for b in combinable: + self.check_src_roundtrip(f"{a}; {b}", mode='single') + + def test_unparse_interactive_integrity_1(self): + # rest of unparse_interactive_integrity tests just make sure mode='single' parse and unparse didn't break + self.check_src_roundtrip( + "if i:\n 'expr'\nelse:\n raise Exception", + "if i:\n 'expr'\nelse:\n raise Exception", + mode='single' + ) + self.check_src_roundtrip( + "@decorator1\n@decorator2\ndef func():\n 'docstring'\n i = 1; 'expr'; raise Exception", + '''@decorator1\n@decorator2\ndef func():\n """docstring"""\n i = 1\n 'expr'\n raise Exception''', + mode='single' + ) + self.check_src_roundtrip( + "@decorator1\n@decorator2\nclass cls:\n 'docstring'\n i = 1; 'expr'; raise Exception", + '''@decorator1\n@decorator2\nclass cls:\n """docstring"""\n i = 1\n 'expr'\n raise Exception''', + mode='single' + ) + + def test_unparse_interactive_integrity_2(self): + for statement in ( + "def x():\n pass", + "def x(y):\n pass", + "async def x():\n pass", + "async def x(y):\n pass", + "for x in y:\n pass", + "async for x in y:\n pass", + "with x():\n pass", + "async with x():\n pass", + "def f():\n pass", + "def f(a):\n pass", + "def f(b=2):\n pass", + "def f(a, b):\n pass", + "def f(a, b=2):\n pass", + "def f(a=5, b=2):\n pass", + "def f(*, a=1, b=2):\n pass", + "def f(*, a=1, b):\n pass", + "def f(*, a, b=2):\n pass", + "def f(a, b=None, *, c, **kwds):\n pass", + "def f(a=2, *args, c=5, d, **kwds):\n pass", + "def f(*args, **kwargs):\n pass", + "class cls:\n\n def f(self):\n pass", + "class cls:\n\n def f(self, a):\n pass", + "class cls:\n\n def f(self, b=2):\n pass", + "class cls:\n\n def f(self, a, b):\n pass", + "class cls:\n\n def f(self, a, b=2):\n pass", + "class cls:\n\n def f(self, a=5, b=2):\n pass", + "class cls:\n\n def f(self, *, a=1, b=2):\n pass", + "class cls:\n\n def f(self, *, a=1, b):\n pass", + "class cls:\n\n def f(self, *, a, b=2):\n pass", + "class cls:\n\n def f(self, a, b=None, *, c, **kwds):\n pass", + "class cls:\n\n def f(self, a=2, *args, c=5, d, **kwds):\n pass", + "class cls:\n\n def f(self, *args, **kwargs):\n pass", + ): + self.check_src_roundtrip(statement, mode='single') + + def test_unparse_interactive_integrity_3(self): + for statement in ( + "def x():", + "def x(y):", + "async def x():", + "async def x(y):", + "for x in y:", + "async for x in y:", + "with x():", + "async with x():", + "def f():", + "def f(a):", + "def f(b=2):", + "def f(a, b):", + "def f(a, b=2):", + "def f(a=5, b=2):", + "def f(*, a=1, b=2):", + "def f(*, a=1, b):", + "def f(*, a, b=2):", + "def f(a, b=None, *, c, **kwds):", + "def f(a=2, *args, c=5, d, **kwds):", + "def f(*args, **kwargs):", + ): + src = statement + '\n i=1;j=2' + out = statement + '\n i = 1\n j = 2' + + self.check_src_roundtrip(src, out, mode='single') + + +class CosmeticTestCase(ASTTestCase): + """Test if there are cosmetic issues caused by unnecessary additions""" + + def test_simple_expressions_parens(self): + self.check_src_roundtrip("(a := b)") + self.check_src_roundtrip("await x") + self.check_src_roundtrip("x if x else y") + self.check_src_roundtrip("lambda x: x") + self.check_src_roundtrip("1 + 1") + self.check_src_roundtrip("1 + 2 / 3") + self.check_src_roundtrip("(1 + 2) / 3") + self.check_src_roundtrip("(1 + 2) * 3 + 4 * (5 + 2)") + self.check_src_roundtrip("(1 + 2) * 3 + 4 * (5 + 2) ** 2") + self.check_src_roundtrip("~x") + self.check_src_roundtrip("x and y") + self.check_src_roundtrip("x and y and z") + self.check_src_roundtrip("x and (y and x)") + self.check_src_roundtrip("(x and y) and z") + self.check_src_roundtrip("(x ** y) ** z ** q") + self.check_src_roundtrip("x >> y") + self.check_src_roundtrip("x << y") + self.check_src_roundtrip("x >> y and x >> z") + self.check_src_roundtrip("x + y - z * q ^ t ** k") + self.check_src_roundtrip("P * V if P and V else n * R * T") + self.check_src_roundtrip("lambda P, V, n: P * V == n * R * T") + self.check_src_roundtrip("flag & (other | foo)") + self.check_src_roundtrip("not x == y") + self.check_src_roundtrip("x == (not y)") + self.check_src_roundtrip("yield x") + self.check_src_roundtrip("yield from x") + self.check_src_roundtrip("call((yield x))") + self.check_src_roundtrip("return x + (yield x)") + + def test_class_bases_and_keywords(self): + self.check_src_roundtrip("class X:\n pass") + self.check_src_roundtrip("class X(A):\n pass") + self.check_src_roundtrip("class X(A, B, C, D):\n pass") + self.check_src_roundtrip("class X(x=y):\n pass") + self.check_src_roundtrip("class X(metaclass=z):\n pass") + self.check_src_roundtrip("class X(x=y, z=d):\n pass") + self.check_src_roundtrip("class X(A, x=y):\n pass") + self.check_src_roundtrip("class X(A, **kw):\n pass") + self.check_src_roundtrip("class X(*args):\n pass") + self.check_src_roundtrip("class X(*args, **kwargs):\n pass") + + def test_fstrings(self): + self.check_src_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\'''') + self.check_src_roundtrip('''f\'-{f\'\'\'*{f"""+{f".{f'{x}'}."}+"""}*\'\'\'}-\'''') + self.check_src_roundtrip('''f\'-{f\'*{f\'\'\'+{f""".{f"{f'{x}'}"}."""}+\'\'\'}*\'}-\'''') + self.check_src_roundtrip('''f"\\u2028{'x'}"''') + self.check_src_roundtrip(r"f'{x}\n'") + self.check_src_roundtrip('''f"{'\\n'}\\n"''') + self.check_src_roundtrip('''f"{f'{x}\\n'}\\n"''') + + def test_docstrings(self): + docstrings = ( + '"""simple doc string"""', + '''"""A more complex one + with some newlines"""''', + '''"""Foo bar baz + + empty newline"""''', + '"""With some \t"""', + '"""Foo "bar" baz """', + '"""\\r"""', + '""""""', + '"""\'\'\'"""', + '"""\'\'\'\'\'\'"""', + '"""🐍⛎𩸽üéş^\\\\X\\\\BB⟿"""', + '"""end in single \'quote\'"""', + "'''end in double \"quote\"'''", + '"""almost end in double "quote"."""', + ) + + for prefix in docstring_prefixes: + for docstring in docstrings: + self.check_src_roundtrip(f"{prefix}{docstring}") + + def test_docstrings_negative_cases(self): + # Test some cases that involve strings in the children of the + # first node but aren't docstrings to make sure we don't have + # False positives. + docstrings_negative = ( + 'a = """false"""', + '"""false""" + """unless its optimized"""', + '1 + 1\n"""false"""', + 'f"""no, top level but f-fstring"""' + ) + for prefix in docstring_prefixes: + for negative in docstrings_negative: + # this cases should be result with single quote + # rather then triple quoted docstring + src = f"{prefix}{negative}" + self.check_ast_roundtrip(src) + self.check_src_dont_roundtrip(src) + + def test_unary_op_factor(self): + for prefix in ("+", "-", "~"): + self.check_src_roundtrip(f"{prefix}1") + for prefix in ("not",): + self.check_src_roundtrip(f"{prefix} 1") + + def test_slices(self): + self.check_src_roundtrip("a[()]") + self.check_src_roundtrip("a[1]") + self.check_src_roundtrip("a[1, 2]") + # Note that `a[*a]`, `a[*a,]`, and `a[(*a,)]` all evaluate to the same + # thing at runtime and have the same AST, but only `a[*a,]` passes + # this test, because that's what `ast.unparse` produces. + self.check_src_roundtrip("a[*a,]") + self.check_src_roundtrip("a[1, *a]") + self.check_src_roundtrip("a[*a, 2]") + self.check_src_roundtrip("a[1, *a, 2]") + self.check_src_roundtrip("a[*a, *a]") + self.check_src_roundtrip("a[1, *a, *a]") + self.check_src_roundtrip("a[*a, 1, *a]") + self.check_src_roundtrip("a[*a, *a, 1]") + self.check_src_roundtrip("a[1, *a, *a, 2]") + self.check_src_roundtrip("a[1:2, *a]") + self.check_src_roundtrip("a[*a, 1:2]") + + def test_lambda_parameters(self): + self.check_src_roundtrip("lambda: something") + self.check_src_roundtrip("four = lambda: 2 + 2") + self.check_src_roundtrip("lambda x: x * 2") + self.check_src_roundtrip("square = lambda n: n ** 2") + self.check_src_roundtrip("lambda x, y: x + y") + self.check_src_roundtrip("add = lambda x, y: x + y") + self.check_src_roundtrip("lambda x, y, /, z, q, *, u: None") + self.check_src_roundtrip("lambda x, *y, **z: None") + + def test_star_expr_assign_target(self): + for source_type, source in [ + ("single assignment", "{target} = foo"), + ("multiple assignment", "{target} = {target} = bar"), + ("for loop", "for {target} in foo:\n pass"), + ("async for loop", "async for {target} in foo:\n pass") + ]: + for target in [ + "a", + "a,", + "a, b", + "a, *b, c", + "a, (b, c), d", + "a, (b, c, d), *e", + "a, (b, *c, d), e", + "a, (b, *c, (d, e), f), g", + "[a]", + "[a, b]", + "[a, *b, c]", + "[a, [b, c], d]", + "[a, [b, c, d], *e]", + "[a, [b, *c, d], e]", + "[a, [b, *c, [d, e], f], g]", + "a, [b, c], d", + "[a, b, (c, d), (e, f)]", + "a, b, [*c], d, e" + ]: + with self.subTest(source_type=source_type, target=target): + self.check_src_roundtrip(source.format(target=target)) + + def test_star_expr_assign_target_multiple(self): + self.check_src_roundtrip("() = []") + self.check_src_roundtrip("[] = ()") + self.check_src_roundtrip("() = [a] = c, = [d] = e, f = () = g = h") + self.check_src_roundtrip("a = b = c = d") + self.check_src_roundtrip("a, b = c, d = e, f = g") + self.check_src_roundtrip("[a, b] = [c, d] = [e, f] = g") + self.check_src_roundtrip("a, b = [c, d] = e, f = g") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_multiquote_joined_string(self): + self.check_ast_roundtrip("f\"'''{1}\\\"\\\"\\\"\" ") + self.check_ast_roundtrip("""f"'''{1}""\\"" """) + self.check_ast_roundtrip("""f'""\"{1}''' """) + self.check_ast_roundtrip("""f'""\"{1}""\\"' """) + + self.check_ast_roundtrip("""f"'''{"\\n"}""\\"" """) + self.check_ast_roundtrip("""f'""\"{"\\n"}''' """) + self.check_ast_roundtrip("""f'""\"{"\\n"}""\\"' """) + + self.check_ast_roundtrip("""f'''""\"''\\'{"\\n"}''' """) + self.check_ast_roundtrip("""f'''""\"''\\'{"\\n\\"'"}''' """) + self.check_ast_roundtrip("""f'''""\"''\\'{""\"\\n\\"'''""\" '''\\n'''}''' """) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxWarning not triggered + def test_backslash_in_format_spec(self): + import re + msg = re.escape('"\\ " is an invalid escape sequence. ' + 'Such sequences will not work in the future. ' + 'Did you mean "\\\\ "? A raw string is also an option.') + with self.assertWarnsRegex(SyntaxWarning, msg): + self.check_ast_roundtrip("""f"{x:\\ }" """) + self.check_ast_roundtrip("""f"{x:\\n}" """) + + self.check_ast_roundtrip("""f"{x:\\\\ }" """) + + with self.assertWarnsRegex(SyntaxWarning, msg): + self.check_ast_roundtrip("""f"{x:\\\\\\ }" """) + self.check_ast_roundtrip("""f"{x:\\\\\\n}" """) + + self.check_ast_roundtrip("""f"{x:\\\\\\\\ }" """) + + def test_quote_in_format_spec(self): + self.check_ast_roundtrip("""f"{x:'}" """) + self.check_ast_roundtrip("""f"{x:\\'}" """) + self.check_ast_roundtrip("""f"{x:\\\\'}" """) + + self.check_ast_roundtrip("""f'\\'{x:"}' """) + self.check_ast_roundtrip("""f'\\'{x:\\"}' """) + self.check_ast_roundtrip("""f'\\'{x:\\\\"}' """) + + def test_type_params(self): + self.check_ast_roundtrip("type A = int") + self.check_ast_roundtrip("type A[T] = int") + self.check_ast_roundtrip("type A[T: int] = int") + self.check_ast_roundtrip("type A[T = int] = int") + self.check_ast_roundtrip("type A[T: int = int] = int") + self.check_ast_roundtrip("type A[**P] = int") + self.check_ast_roundtrip("type A[**P = int] = int") + self.check_ast_roundtrip("type A[*Ts] = int") + self.check_ast_roundtrip("type A[*Ts = int] = int") + self.check_ast_roundtrip("type A[*Ts = *int] = int") + self.check_ast_roundtrip("def f[T: int = int, **P = int, *Ts = *int]():\n pass") + self.check_ast_roundtrip("class C[T: int = int, **P = int, *Ts = *int]():\n pass") + + +class ManualASTCreationTestCase(unittest.TestCase): + """Test that AST nodes created without a type_params field unparse correctly.""" + + def test_class(self): + node = ast.ClassDef(name="X", bases=[], keywords=[], body=[ast.Pass()], decorator_list=[]) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "class X:\n pass") + + def test_class_with_type_params(self): + node = ast.ClassDef(name="X", bases=[], keywords=[], body=[ast.Pass()], decorator_list=[], + type_params=[ast.TypeVar("T")]) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "class X[T]:\n pass") + + def test_function(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f():\n pass") + + def test_function_with_type_params(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + type_params=[ast.TypeVar("T")], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f[T]():\n pass") + + def test_function_with_type_params_and_bound(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + type_params=[ast.TypeVar("T", bound=ast.Name("int", ctx=ast.Load()))], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f[T: int]():\n pass") + + def test_function_with_type_params_and_default(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(), + body=[ast.Pass()], + type_params=[ + ast.TypeVar("T", default_value=ast.Constant(value=1)), + ast.TypeVarTuple("Ts", default_value=ast.Starred(value=ast.Constant(value=1), ctx=ast.Load())), + ast.ParamSpec("P", default_value=ast.Constant(value=1)), + ], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f[T = 1, *Ts = *1, **P = 1]():\n pass") + + def test_async_function(self): + node = ast.AsyncFunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "async def f():\n pass") + + def test_async_function_with_type_params(self): + node = ast.AsyncFunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + type_params=[ast.TypeVar("T")], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "async def f[T]():\n pass") + + def test_async_function_with_type_params_and_default(self): + node = ast.AsyncFunctionDef( + name="f", + args=ast.arguments(), + body=[ast.Pass()], + type_params=[ + ast.TypeVar("T", default_value=ast.Constant(value=1)), + ast.TypeVarTuple("Ts", default_value=ast.Starred(value=ast.Constant(value=1), ctx=ast.Load())), + ast.ParamSpec("P", default_value=ast.Constant(value=1)), + ], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "async def f[T = 1, *Ts = *1, **P = 1]():\n pass") + + +class DirectoryTestCase(ASTTestCase): + """Test roundtrip behaviour on all files in Lib and Lib/test.""" + + lib_dir = pathlib.Path(__file__).parent / ".." + test_directories = (lib_dir, lib_dir / "test") + run_always_files = {"test_grammar.py", "test_syntax.py", "test_compile.py", + "test_ast.py", "test_asdl_parser.py", "test_fstring.py", + "test_patma.py", "test_type_alias.py", "test_type_params.py", + "test_tokenize.py", "test_tstring.py"} + + _files_to_test = None + + @classmethod + def files_to_test(cls): + + if cls._files_to_test is not None: + return cls._files_to_test + + items = [ + item.resolve() + for directory in cls.test_directories + for item in directory.glob("*.py") + if not item.name.startswith("bad") + ] + + # Test limited subset of files unless the 'cpu' resource is specified. + if not test.support.is_resource_enabled("cpu"): + + tests_to_run_always = {item for item in items if + item.name in cls.run_always_files} + + items = set(random.sample(items, 10)) + + # Make sure that at least tests that heavily use grammar features are + # always considered in order to reduce the chance of missing something. + items = list(items | tests_to_run_always) + + # bpo-31174: Store the names sample to always test the same files. + # It prevents false alarms when hunting reference leaks. + cls._files_to_test = items + + return items + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_files(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', SyntaxWarning) + + for item in self.files_to_test(): + if test.support.verbose: + print(f"Testing {item.absolute()}") + + with self.subTest(filename=item): + source = read_pyfile(item) + self.check_ast_roundtrip(source) + + +if __name__ == "__main__": + unittest.main() From ff49bfe3a72a7ba248b544b96a96735271462221 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 2 Feb 2026 14:57:56 +0900 Subject: [PATCH 6/8] mark ast & test_genericalias --- Lib/test/test_ast/test_ast.py | 63 ++++++++++++-------------- Lib/test/test_exception_group.py | 2 - Lib/test/test_genericalias.py | 2 - crates/codegen/src/unparse.rs | 4 +- crates/vm/src/stdlib/ast/expression.rs | 23 +++++++--- crates/vm/src/stdlib/ast/parameter.rs | 3 +- crates/vm/src/stdlib/ast/python.rs | 5 +- crates/vm/src/stdlib/ast/statement.rs | 9 ++-- 8 files changed, 53 insertions(+), 58 deletions(-) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 9100cf44335..f59090dec7c 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -97,7 +97,7 @@ def test_AST_objects(self): # "ast.AST constructor takes 0 positional arguments" ast.AST(2) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "type object 'ast.AST' has no attribute '_fields'" does not match "'AST' object has no attribute '_fields'" def test_AST_fields_NULL_check(self): # See: https://github.com/python/cpython/issues/126105 old_value = ast.AST._fields @@ -115,7 +115,7 @@ def cleanup(): with self.assertRaisesRegex(AttributeError, msg): ast.AST() - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: .X object at 0x7e85c3a80> is not None def test_AST_garbage_collection(self): class X: pass @@ -140,7 +140,7 @@ def test_snippets(self): with self.subTest(action="compiling", input=i, kind=kind): compile(ast_tree, "?", kind) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got <_ast.TemplateStr object at 0x7e85c34e0> def test_ast_validation(self): # compile() is the only function that calls PyAST_Validate snippets_to_validate = exec_tests + single_tests + eval_tests @@ -439,7 +439,7 @@ def _construct_ast_class(self, cls): kwargs[name] = self._construct_ast_class(typ) return cls(**kwargs) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: type object 'arguments' has no attribute '__annotations__' def test_arguments(self): x = ast.arguments() self.assertEqual(x._fields, ('posonlyargs', 'args', 'vararg', 'kwonlyargs', @@ -590,7 +590,7 @@ def test_no_fields(self): x = ast.Sub() self.assertEqual(x._fields, ()) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'but got expr()' not found in 'expected some sort of expr, but got <_ast.expr object at 0x7e911a9a0>' def test_invalid_sum(self): pos = dict(lineno=2, col_offset=3) m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) @@ -598,7 +598,7 @@ def test_invalid_sum(self): compile(m, "", "exec") self.assertIn("but got expr()", str(cm.exception)) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: expected str for name def test_invalid_identifier(self): m = ast.Module([ast.Expr(ast.Name(42, ast.Load()))], []) ast.fix_missing_locations(m) @@ -615,7 +615,7 @@ def test_invalid_constant(self): ): compile(e, "", "eval") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None def test_empty_yield_from(self): # Issue 16546: yield from value is not optional. empty_yield_from = ast.parse("def f():\n yield from g()") @@ -670,7 +670,7 @@ def test_issue39579_dotted_name_end_col_offset(self): attr_b = tree.body[0].decorator_list[0].value self.assertEqual(attr_b.end_col_offset, 4) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: None != 'withitem(expr context_expr, expr? optional_vars)' def test_ast_asdl_signature(self): self.assertEqual(ast.withitem.__doc__, "withitem(expr context_expr, expr? optional_vars)") self.assertEqual(ast.GtE.__doc__, "GtE") @@ -776,7 +776,6 @@ def test_compare_fieldless(self): del a2.id self.assertTrue(ast.compare(a1, a2)) - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: type object '_ast.Module' has no attribute '_field_types' def test_compare_modes(self): for mode, sources in ( ("exec", exec_tests), @@ -1100,10 +1099,6 @@ def test_tstring(self): self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_classattrs_deprecated(self): - return super().test_classattrs_deprecated() - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags def test_optimization_levels_const_folding(self): return super().test_optimization_levels_const_folding() @@ -1472,7 +1467,6 @@ def test_replace_reject_unknown_instance_fields(self): class ASTHelpers_Test(unittest.TestCase): maxDiff = None - @unittest.expectedFailure # TODO: RUSTPYTHON def test_parse(self): a = ast.parse('foo(1 + 1)') b = compile('foo(1 + 1)', '', 'exec', ast.PyCF_ONLY_AST) @@ -1486,7 +1480,7 @@ def test_parse_in_error(self): ast.literal_eval(r"'\U'") self.assertIsNotNone(e.exception.__context__) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; + Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))]) def test_dump(self): node = ast.parse('spam(eggs, "and cheese")') self.assertEqual(ast.dump(node), @@ -1507,7 +1501,7 @@ def test_dump(self): "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])" ) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; - type_ignores=[]) def test_dump_indent(self): node = ast.parse('spam(eggs, "and cheese")') self.assertEqual(ast.dump(node, indent=3), """\ @@ -1563,7 +1557,7 @@ def test_dump_indent(self): end_lineno=1, end_col_offset=24)])""") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; + Raise() def test_dump_incomplete(self): node = ast.Raise(lineno=3, col_offset=4) self.assertEqual(ast.dump(node), @@ -1731,7 +1725,7 @@ def check_text(code, empty, full, **kwargs): full="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)], type_ignores=[])", ) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^^^^^^ ^^^^^^^^^ def test_copy_location(self): src = ast.parse('1 + 1', mode='eval') src.body.right = ast.copy_location(ast.Constant(2), src.body.right) @@ -1749,7 +1743,7 @@ def test_copy_location(self): self.assertEqual(new.lineno, 1) self.assertEqual(new.col_offset, 1) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; + Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], lineno=1, col_offset=0, end_lineno=1, end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, end_col_offset=13), Expr(value=Call(func=Name(id='spam', ctx=Load(), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)], lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)]) def test_fix_missing_locations(self): src = ast.parse('write("spam")') src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()), @@ -1769,7 +1763,7 @@ def test_fix_missing_locations(self): "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])" ) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ def test_increment_lineno(self): src = ast.parse('1 + 1', mode='eval') self.assertEqual(ast.increment_lineno(src, n=3), src) @@ -1813,7 +1807,7 @@ def test_iter_fields(self): self.assertEqual(d.pop('func').id, 'foo') self.assertEqual(d, {'keywords': [], 'args': []}) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; + keyword(arg='eggs', value=Constant(value='leek')) def test_iter_child_nodes(self): node = ast.parse("spam(23, 42, eggs='leek')", mode='eval') self.assertEqual(len(list(ast.iter_child_nodes(node.body))), 4) @@ -1966,7 +1960,7 @@ def test_literal_eval_malformed_dict_nodes(self): malformed = ast.Dict(keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)]) self.assertRaises(ValueError, ast.literal_eval, malformed) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: expected an expression def test_literal_eval_trailing_ws(self): self.assertEqual(ast.literal_eval(" -1"), -1) self.assertEqual(ast.literal_eval("\t\t-1"), -1) @@ -1985,7 +1979,7 @@ def test_literal_eval_malformed_lineno(self): with self.assertRaisesRegex(ValueError, msg): ast.literal_eval(node) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "unexpected indent" does not match "expected an expression (, line 2)" def test_literal_eval_syntax_errors(self): with self.assertRaisesRegex(SyntaxError, "unexpected indent"): ast.literal_eval(r''' @@ -1993,7 +1987,7 @@ def test_literal_eval_syntax_errors(self): (\ \ ''') - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: required field "lineno" missing from alias def test_bad_integer(self): # issue13436: Bad error message with invalid numeric values body = [ast.ImportFrom(module='time', @@ -2222,7 +2216,7 @@ def test_if(self): [ast.Expr(ast.Name("x", ast.Store()))]) self.stmt(i, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: empty items on With def test_with(self): p = ast.Pass() self.stmt(ast.With([], [p]), "empty items on With") @@ -2263,7 +2257,7 @@ def test_try(self): t = ast.Try([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) self.stmt(t, "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised def test_try_star(self): p = ast.Pass() t = ast.TryStar([], [], [], [p]) @@ -2296,7 +2290,7 @@ def test_assert(self): def test_import(self): self.stmt(ast.Import([]), "empty names on Import") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; OverflowError: Python int too large to convert to Rust u32 def test_importfrom(self): imp = ast.ImportFrom(None, [ast.alias("x", None)], -42) self.stmt(imp, "Negative ImportFrom level") @@ -2354,7 +2348,7 @@ def test_dict(self): d = ast.Dict([ast.Name("x", ast.Load())], [None]) self.expr(d, "None disallowed") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None def test_set(self): self.expr(ast.Set([None]), "None disallowed") s = ast.Set([ast.Name("x", ast.Store())]) @@ -2412,7 +2406,7 @@ def factory(comps): return ast.DictComp(k, v, comps) self._check_comprehension(factory) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: 'yield' outside function def test_yield(self): self.expr(ast.Yield(ast.Name("x", ast.Store())), "must have Load") self.expr(ast.YieldFrom(ast.Name("x", ast.Store())), "must have Load") @@ -2478,11 +2472,11 @@ def _sequence(self, fac): self.expr(fac([ast.Name("x", ast.Store())], ast.Load()), "must have Load context") - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None def test_list(self): self._sequence(ast.List) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None def test_tuple(self): self._sequence(ast.Tuple) @@ -3264,7 +3258,7 @@ def visit_Call(self, node: ast.Call): class ASTConstructorTests(unittest.TestCase): """Test the autogenerated constructors for AST nodes.""" - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: DeprecationWarning not triggered def test_FunctionDef(self): args = ast.arguments() self.assertEqual(args.args, []) @@ -3278,7 +3272,7 @@ def test_FunctionDef(self): self.assertEqual(node.name, 'foo') self.assertEqual(node.decorator_list, []) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: None is not an instance of def test_expr_context(self): name = ast.Name("x") self.assertEqual(name.id, "x") @@ -3382,7 +3376,7 @@ class BadFields(ast.AST): with self.assertWarnsRegex(DeprecationWarning, r"Field b'\\xff\\xff.*' .*"): obj = BadFields() - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: None != [] def test_complete_field_types(self): class _AllFieldTypes(ast.AST): _fields = ('a', 'b') @@ -3508,7 +3502,6 @@ def check_output(self, source, expect, *flags): expect = self.text_normalize(expect) self.assertEqual(res, expect) - @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_resource('cpu') def test_invocation(self): # test various combinations of parameters diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 2b48530a309..453bc12234e 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -9,8 +9,6 @@ def test_exception_group_types(self): self.assertTrue(issubclass(ExceptionGroup, BaseExceptionGroup)) self.assertTrue(issubclass(BaseExceptionGroup, BaseException)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_is_not_generic_type(self): with self.assertRaisesRegex(TypeError, 'Exception'): Exception[OSError] diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index c1c49dc29d8..3da8c2b1eea 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -151,7 +151,6 @@ def test_subscriptable(self): self.assertEqual(alias.__args__, (int,)) self.assertEqual(alias.__parameters__, ()) - @unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message def test_unsubscriptable(self): for t in int, str, float, Sized, Hashable: tname = t.__name__ @@ -365,7 +364,6 @@ def test_type_generic(self): self.assertEqual(t(test), Test) self.assertEqual(t(0), int) - @unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message def test_type_subclass_generic(self): class MyType(type): pass diff --git a/crates/codegen/src/unparse.rs b/crates/codegen/src/unparse.rs index cb9f3783fc3..1e958659dc4 100644 --- a/crates/codegen/src/unparse.rs +++ b/crates/codegen/src/unparse.rs @@ -363,9 +363,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.p(")")?; } ast::Expr::FString(ast::ExprFString { value, .. }) => self.unparse_fstring(value)?, - ast::Expr::TString(ast::ExprTString { value, .. }) => { - self.unparse_tstring(value)? - } + ast::Expr::TString(ast::ExprTString { value, .. }) => self.unparse_tstring(value)?, ast::Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) => { if value.is_unicode() { self.p("u")? diff --git a/crates/vm/src/stdlib/ast/expression.rs b/crates/vm/src/stdlib/ast/expression.rs index fc1831bf597..e60fe18b73f 100644 --- a/crates/vm/src/stdlib/ast/expression.rs +++ b/crates/vm/src/stdlib/ast/expression.rs @@ -336,18 +336,27 @@ impl Node for ast::ExprLambda { .into_ref_with_type(vm, pyast::NodeArguments::static_type().to_owned()) .unwrap(); let args_dict = args_node.as_object().dict().unwrap(); - args_dict.set_item("posonlyargs", vm.ctx.new_list(vec![]).into(), vm).unwrap(); - args_dict.set_item("args", vm.ctx.new_list(vec![]).into(), vm).unwrap(); + args_dict + .set_item("posonlyargs", vm.ctx.new_list(vec![]).into(), vm) + .unwrap(); + args_dict + .set_item("args", vm.ctx.new_list(vec![]).into(), vm) + .unwrap(); args_dict.set_item("vararg", vm.ctx.none(), vm).unwrap(); - args_dict.set_item("kwonlyargs", vm.ctx.new_list(vec![]).into(), vm).unwrap(); - args_dict.set_item("kw_defaults", vm.ctx.new_list(vec![]).into(), vm).unwrap(); + args_dict + .set_item("kwonlyargs", vm.ctx.new_list(vec![]).into(), vm) + .unwrap(); + args_dict + .set_item("kw_defaults", vm.ctx.new_list(vec![]).into(), vm) + .unwrap(); args_dict.set_item("kwarg", vm.ctx.none(), vm).unwrap(); - args_dict.set_item("defaults", vm.ctx.new_list(vec![]).into(), vm).unwrap(); + args_dict + .set_item("defaults", vm.ctx.new_list(vec![]).into(), vm) + .unwrap(); args_node.into() } }; - dict.set_item("args", args, vm) - .unwrap(); + dict.set_item("args", args, vm).unwrap(); dict.set_item("body", body.ast_to_object(vm, source_file), vm) .unwrap(); node_add_location(&dict, _range, vm, source_file); diff --git a/crates/vm/src/stdlib/ast/parameter.rs b/crates/vm/src/stdlib/ast/parameter.rs index b1942d833ba..dc4f32203ca 100644 --- a/crates/vm/src/stdlib/ast/parameter.rs +++ b/crates/vm/src/stdlib/ast/parameter.rs @@ -127,8 +127,7 @@ impl Node for ast::Parameter { ) .unwrap(); // Ruff AST doesn't track type_comment, so always set to None - dict.set_item("type_comment", _vm.ctx.none(), _vm) - .unwrap(); + dict.set_item("type_comment", _vm.ctx.none(), _vm).unwrap(); node_add_location(&dict, range, _vm, source_file); node.into() } diff --git a/crates/vm/src/stdlib/ast/python.rs b/crates/vm/src/stdlib/ast/python.rs index 924026735d7..e973be22e13 100644 --- a/crates/vm/src/stdlib/ast/python.rs +++ b/crates/vm/src/stdlib/ast/python.rs @@ -89,7 +89,10 @@ pub(crate) mod _ast { // Set default values only for built-in AST nodes (_field_types present). // Custom AST subclasses without _field_types do NOT get automatic defaults. - let has_field_types = zelf.class().get_attr(vm.ctx.intern_str("_field_types")).is_some(); + let has_field_types = zelf + .class() + .get_attr(vm.ctx.intern_str("_field_types")) + .is_some(); if has_field_types { // ASDL list fields (type*) default to empty list, // optional fields (type?) default to None. diff --git a/crates/vm/src/stdlib/ast/statement.rs b/crates/vm/src/stdlib/ast/statement.rs index 620a8317878..1d8f1cbcf00 100644 --- a/crates/vm/src/stdlib/ast/statement.rs +++ b/crates/vm/src/stdlib/ast/statement.rs @@ -183,8 +183,7 @@ impl Node for ast::StmtFunctionDef { dict.set_item("returns", returns.ast_to_object(vm, source_file), vm) .unwrap(); // Ruff AST doesn't track type_comment, so always set to None - dict.set_item("type_comment", vm.ctx.none(), vm) - .unwrap(); + dict.set_item("type_comment", vm.ctx.none(), vm).unwrap(); dict.set_item( "type_params", type_params @@ -648,8 +647,7 @@ impl Node for ast::StmtFor { dict.set_item("orelse", orelse.ast_to_object(_vm, source_file), _vm) .unwrap(); // Ruff AST doesn't track type_comment, so always set to None - dict.set_item("type_comment", _vm.ctx.none(), _vm) - .unwrap(); + dict.set_item("type_comment", _vm.ctx.none(), _vm).unwrap(); node_add_location(&dict, _range, _vm, source_file); node.into() } @@ -801,8 +799,7 @@ impl Node for ast::StmtWith { dict.set_item("body", body.ast_to_object(_vm, source_file), _vm) .unwrap(); // Ruff AST doesn't track type_comment, so always set to None - dict.set_item("type_comment", _vm.ctx.none(), _vm) - .unwrap(); + dict.set_item("type_comment", _vm.ctx.none(), _vm).unwrap(); node_add_location(&dict, _range, _vm, source_file); node.into() } From 00cdb307e39ad8c852d6d70fe28dbec679757cf7 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 2 Feb 2026 19:42:58 +0900 Subject: [PATCH 7/8] Fix AST field defaults and compile() type check - Extract empty_arguments_object helper from expression.rs - Fix LIST_FIELDS ambiguity: "args" and "body" have different ASDL types per node (e.g. Lambda.args is `arguments`, not `expr*`; Lambda.body is `expr`, not `stmt*`) - Replace class name string comparison in compile() with fast_isinstance to accept AST subclasses --- Lib/test/test_genericclass.py | 1 - crates/vm/src/protocol/object.rs | 1 + crates/vm/src/stdlib/ast.rs | 36 +++++++++++++++++++++++ crates/vm/src/stdlib/ast/expression.rs | 26 +---------------- crates/vm/src/stdlib/ast/python.rs | 31 +++++++++++++++----- crates/vm/src/stdlib/builtins.rs | 40 +++++++++++++++----------- 6 files changed, 86 insertions(+), 49 deletions(-) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py index 498904dd97f..e530b463966 100644 --- a/Lib/test/test_genericclass.py +++ b/Lib/test/test_genericclass.py @@ -228,7 +228,6 @@ def __class_getitem__(cls, one, two): with self.assertRaises(TypeError): C_too_many[int] - @unittest.expectedFailure # TODO: RUSTPYTHON def test_class_getitem_errors_2(self): class C: def __class_getitem__(cls, item): diff --git a/crates/vm/src/protocol/object.rs b/crates/vm/src/protocol/object.rs index 02a712979f2..9c4dcb043f7 100644 --- a/crates/vm/src/protocol/object.rs +++ b/crates/vm/src/protocol/object.rs @@ -704,6 +704,7 @@ impl PyObject { if let Some(class_getitem) = vm.get_attribute_opt(self.to_owned(), identifier!(vm, __class_getitem__))? + && !vm.is_none(&class_getitem) { return class_getitem.call((needle,), vm); } diff --git a/crates/vm/src/stdlib/ast.rs b/crates/vm/src/stdlib/ast.rs index cb7bfc289b0..bdf90811259 100644 --- a/crates/vm/src/stdlib/ast.rs +++ b/crates/vm/src/stdlib/ast.rs @@ -265,6 +265,42 @@ fn node_add_location( .unwrap(); } +/// Return the expected AST mod type class for a compile() mode string. +pub(crate) fn mode_type_and_name( + ctx: &Context, + mode: &str, +) -> Option<(PyRef, &'static str)> { + match mode { + "exec" => Some((pyast::NodeModModule::make_class(ctx), "Module")), + "eval" => Some((pyast::NodeModExpression::make_class(ctx), "Expression")), + "single" => Some((pyast::NodeModInteractive::make_class(ctx), "Interactive")), + "func_type" => Some((pyast::NodeModFunctionType::make_class(ctx), "FunctionType")), + _ => None, + } +} + +/// Create an empty `arguments` AST node (no parameters). +fn empty_arguments_object(vm: &VirtualMachine) -> PyObjectRef { + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeArguments::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + for list_field in [ + "posonlyargs", + "args", + "kwonlyargs", + "kw_defaults", + "defaults", + ] { + dict.set_item(list_field, vm.ctx.new_list(vec![]).into(), vm) + .unwrap(); + } + for none_field in ["vararg", "kwarg"] { + dict.set_item(none_field, vm.ctx.none(), vm).unwrap(); + } + node.into() +} + #[cfg(feature = "parser")] pub(crate) fn parse( vm: &VirtualMachine, diff --git a/crates/vm/src/stdlib/ast/expression.rs b/crates/vm/src/stdlib/ast/expression.rs index e60fe18b73f..ebfa471c842 100644 --- a/crates/vm/src/stdlib/ast/expression.rs +++ b/crates/vm/src/stdlib/ast/expression.rs @@ -330,31 +330,7 @@ impl Node for ast::ExprLambda { // Lambda with no parameters should have an empty arguments object, not None let args = match parameters { Some(params) => params.ast_to_object(vm, source_file), - None => { - // Create an empty arguments object - let args_node = NodeAst - .into_ref_with_type(vm, pyast::NodeArguments::static_type().to_owned()) - .unwrap(); - let args_dict = args_node.as_object().dict().unwrap(); - args_dict - .set_item("posonlyargs", vm.ctx.new_list(vec![]).into(), vm) - .unwrap(); - args_dict - .set_item("args", vm.ctx.new_list(vec![]).into(), vm) - .unwrap(); - args_dict.set_item("vararg", vm.ctx.none(), vm).unwrap(); - args_dict - .set_item("kwonlyargs", vm.ctx.new_list(vec![]).into(), vm) - .unwrap(); - args_dict - .set_item("kw_defaults", vm.ctx.new_list(vec![]).into(), vm) - .unwrap(); - args_dict.set_item("kwarg", vm.ctx.none(), vm).unwrap(); - args_dict - .set_item("defaults", vm.ctx.new_list(vec![]).into(), vm) - .unwrap(); - args_node.into() - } + None => empty_arguments_object(vm), }; dict.set_item("args", args, vm).unwrap(); dict.set_item("body", body.ast_to_object(vm, source_file), vm) diff --git a/crates/vm/src/stdlib/ast/python.rs b/crates/vm/src/stdlib/ast/python.rs index e973be22e13..5bea02c641d 100644 --- a/crates/vm/src/stdlib/ast/python.rs +++ b/crates/vm/src/stdlib/ast/python.rs @@ -95,12 +95,11 @@ pub(crate) mod _ast { .is_some(); if has_field_types { // ASDL list fields (type*) default to empty list, - // optional fields (type?) default to None. + // optional/required fields default to None. + // Fields that are always list-typed regardless of node class. const LIST_FIELDS: &[&str] = &[ - "args", "argtypes", "bases", - "body", "cases", "comparators", "decorator_list", @@ -113,11 +112,13 @@ pub(crate) mod _ast { "items", "keys", "kw_defaults", + "kwd_attrs", + "kwd_patterns", "keywords", "kwonlyargs", "names", - "orelse", "ops", + "patterns", "posonlyargs", "targets", "type_ignores", @@ -125,19 +126,35 @@ pub(crate) mod _ast { "values", ]; + let class_name = zelf.class().name().to_string(); + for field in &fields { if !set_fields.contains(field.as_str()) { - let default: PyObjectRef = if LIST_FIELDS.contains(&field.as_str()) { + let field_name = field.as_str(); + // Some field names have different ASDL types depending on the node. + // For example, "args" is `expr*` in Call but `arguments` in Lambda. + // "body" and "orelse" are `stmt*` in most nodes but `expr` in IfExp. + let is_list_field = if field_name == "args" { + class_name == "Call" || class_name == "arguments" + } else if field_name == "body" || field_name == "orelse" { + !matches!( + class_name.as_str(), + "Lambda" | "Expression" | "IfExp" + ) + } else { + LIST_FIELDS.contains(&field_name) + }; + + let default: PyObjectRef = if is_list_field { vm.ctx.new_list(vec![]).into() } else { vm.ctx.none() }; - zelf.set_attr(vm.ctx.intern_str(field.as_str()), default, vm)?; + zelf.set_attr(vm.ctx.intern_str(field_name), default, vm)?; } } // Special defaults that are not None or empty list - let class_name = &*zelf.class().name(); if class_name == "ImportFrom" && !set_fields.contains("level") { zelf.set_attr("level", vm.ctx.new_int(0), vm)?; } diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index 8bfbffcc613..ee9c26e3317 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -142,24 +142,32 @@ mod builtins { .fast_isinstance(&ast::NodeAst::make_class(&vm.ctx)) { use num_traits::Zero; - let flags = args.flags.map_or(Ok(0), |v| v.try_to_primitive(vm))?; + let flags: i32 = args.flags.map_or(Ok(0), |v| v.try_to_primitive(vm))?; + let is_ast_only = !(flags & ast::PY_COMPILE_FLAG_AST_ONLY).is_zero(); + + // func_type mode requires PyCF_ONLY_AST + if mode_str == "func_type" && !is_ast_only { + return Err(vm.new_value_error( + "compile() mode 'func_type' requires flag PyCF_ONLY_AST".to_owned(), + )); + } + // compile(ast_node, ..., PyCF_ONLY_AST) returns the AST after validation - if !(flags & ast::PY_COMPILE_FLAG_AST_ONLY).is_zero() { - let expected_type = match mode_str { - "exec" => "Module", - "eval" => "Expression", - "single" => "Interactive", - "func_type" => "FunctionType", - _ => { - return Err(vm.new_value_error(format!( - "compile() mode must be 'exec', 'eval', 'single' or 'func_type', got '{mode_str}'" - ))); - } - }; - let cls_name = args.source.class().name().to_string(); - if cls_name != expected_type { + if is_ast_only { + let (expected_type, expected_name) = ast::mode_type_and_name( + &vm.ctx, mode_str, + ) + .ok_or_else(|| { + vm.new_value_error( + "compile() mode must be 'exec', 'eval', 'single' or 'func_type'" + .to_owned(), + ) + })?; + if !args.source.fast_isinstance(&expected_type) { return Err(vm.new_type_error(format!( - "expected {expected_type} node, got {cls_name}" + "expected {} node, got {}", + expected_name, + args.source.class().name() ))); } return Ok(args.source); From 9e7faba7f3e3e4c86815c8a3f40b191e39d4920a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 3 Feb 2026 05:59:58 +0000 Subject: [PATCH 8/8] Auto-format: cargo fmt --all --- crates/vm/src/protocol/object.rs | 2 +- crates/vm/src/stdlib/ast/python.rs | 5 +---- crates/vm/src/stdlib/builtins.rs | 16 +++++++--------- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/crates/vm/src/protocol/object.rs b/crates/vm/src/protocol/object.rs index 9c4dcb043f7..d0e068e5073 100644 --- a/crates/vm/src/protocol/object.rs +++ b/crates/vm/src/protocol/object.rs @@ -704,7 +704,7 @@ impl PyObject { if let Some(class_getitem) = vm.get_attribute_opt(self.to_owned(), identifier!(vm, __class_getitem__))? - && !vm.is_none(&class_getitem) + && !vm.is_none(&class_getitem) { return class_getitem.call((needle,), vm); } diff --git a/crates/vm/src/stdlib/ast/python.rs b/crates/vm/src/stdlib/ast/python.rs index 5bea02c641d..17062c99a0d 100644 --- a/crates/vm/src/stdlib/ast/python.rs +++ b/crates/vm/src/stdlib/ast/python.rs @@ -137,10 +137,7 @@ pub(crate) mod _ast { let is_list_field = if field_name == "args" { class_name == "Call" || class_name == "arguments" } else if field_name == "body" || field_name == "orelse" { - !matches!( - class_name.as_str(), - "Lambda" | "Expression" | "IfExp" - ) + !matches!(class_name.as_str(), "Lambda" | "Expression" | "IfExp") } else { LIST_FIELDS.contains(&field_name) }; diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index ee9c26e3317..c9e7d0e9f32 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -154,15 +154,13 @@ mod builtins { // compile(ast_node, ..., PyCF_ONLY_AST) returns the AST after validation if is_ast_only { - let (expected_type, expected_name) = ast::mode_type_and_name( - &vm.ctx, mode_str, - ) - .ok_or_else(|| { - vm.new_value_error( - "compile() mode must be 'exec', 'eval', 'single' or 'func_type'" - .to_owned(), - ) - })?; + let (expected_type, expected_name) = ast::mode_type_and_name(&vm.ctx, mode_str) + .ok_or_else(|| { + vm.new_value_error( + "compile() mode must be 'exec', 'eval', 'single' or 'func_type'" + .to_owned(), + ) + })?; if !args.source.fast_isinstance(&expected_type) { return Err(vm.new_type_error(format!( "expected {} node, got {}",