diff --git a/Cargo.lock b/Cargo.lock index 88146885d85..62ff228be62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,6 +143,36 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "attribute-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0053e96dd3bec5b4879c23a138d6ef26f2cb936c9cdc96274ac2b9ed44b5bb54" +dependencies = [ + "attribute-derive-macro", + "derive-where", + "manyhow", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "attribute-derive-macro" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "463b53ad0fd5b460af4b1915fe045ff4d946d025fb6c4dc3337752eaa980f71b" +dependencies = [ + "collection_literals", + "interpolator", + "manyhow", + "proc-macro-utils", + "proc-macro2", + "quote", + "quote-use", + "syn", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -381,24 +411,16 @@ dependencies = [ ] [[package]] -name = "colorchoice" -version = "1.0.4" +name = "collection_literals" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +checksum = "26b3f65b8fb8e88ba339f7d23a390fe1b0896217da05e2a66c584c9b29a91df8" [[package]] -name = "compact_str" -version = "0.8.1" +name = "colorchoice" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" -dependencies = [ - "castaway", - "cfg-if", - "itoa", - "rustversion", - "ryu", - "static_assertions", -] +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "compact_str" @@ -527,7 +549,7 @@ dependencies = [ "cranelift-entity", "cranelift-isle", "gimli", - "hashbrown", + "hashbrown 0.15.5", "log", "regalloc2", "rustc-hash", @@ -729,6 +751,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "derive-where" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef941ded77d15ca19b40374869ac6000af1c9f2a4c0f3d4c70926287e6364a8f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "digest" version = "0.10.7" @@ -961,6 +994,29 @@ dependencies = [ "version_check", ] +[[package]] +name = "get-size-derive2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a443e77201a230c25f0c11574e9b20e5705f749520e0f30ab0d0974fb1a794" +dependencies = [ + "attribute-derive", + "quote", + "syn", +] + +[[package]] +name = "get-size2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0594e2a78d082f2f8b1615c728391c6a5277f6c017474a7249934fc735945d55" +dependencies = [ + "compact_str", + "get-size-derive2", + "hashbrown 0.16.0", + "smallvec", +] + [[package]] name = "gethostname" version = "1.0.2" @@ -1041,6 +1097,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashbrown" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" + [[package]] name = "heck" version = "0.5.0" @@ -1105,7 +1167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92119844f513ffa41556430369ab02c295a3578af21cf945caa3e9e0c2481ac3" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.5", ] [[package]] @@ -1125,6 +1187,12 @@ dependencies = [ "similar", ] +[[package]] +name = "interpolator" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" + [[package]] name = "is-macro" version = "0.3.7" @@ -1406,7 +1474,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c738d3789301e957a8f7519318fcbb1b92bb95863b28f6938ae5a05be6259f34" dependencies = [ - "hashbrown", + "hashbrown 0.15.5", "itertools 0.14.0", "libm", "ryu", @@ -1448,6 +1516,29 @@ dependencies = [ "malachite-nz", ] +[[package]] +name = "manyhow" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587" +dependencies = [ + "manyhow-macros", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "manyhow-macros" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46fce34d199b78b6e6073abf984c9cf5fd3e9330145a93ee0738a7443e371495" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", +] + [[package]] name = "maplit" version = "1.0.2" @@ -1855,6 +1946,17 @@ dependencies = [ "syn", ] +[[package]] +name = "proc-macro-utils" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071" +dependencies = [ + "proc-macro2", + "quote", + "smallvec", +] + [[package]] name = "proc-macro2" version = "1.0.101" @@ -1943,6 +2045,28 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "quote-use" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9619db1197b497a36178cfc736dc96b271fe918875fbf1344c436a7e93d0321e" +dependencies = [ + "quote", + "quote-use-macros", +] + +[[package]] +name = "quote-use-macros" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82ebfb7faafadc06a7ab141a6f67bcfb24cb8beb158c6fe933f2f035afa99f35" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "r-efi" version = "5.3.0" @@ -2081,7 +2205,7 @@ checksum = "dc06e6b318142614e4a48bc725abbf08ff166694835c43c9dae5a9009704639a" dependencies = [ "allocator-api2", "bumpalo", - "hashbrown", + "hashbrown 0.15.5", "log", "rustc-hash", "smallvec", @@ -2152,50 +2276,58 @@ dependencies = [ [[package]] name = "ruff_python_ast" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.13.1#706be0a6e7e09936511198f2ff8982915520d138" dependencies = [ "aho-corasick", "bitflags 2.9.4", - "compact_str 0.8.1", + "compact_str", + "get-size2", "is-macro", "itertools 0.14.0", "memchr", - "ruff_python_trivia 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.11.0)", - "ruff_source_file 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.11.0)", - "ruff_text_size 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.11.0)", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", "rustc-hash", + "thiserror 2.0.16", ] [[package]] -name = "ruff_python_ast" +name = "ruff_python_codegen" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.13.1#706be0a6e7e09936511198f2ff8982915520d138" +dependencies = [ + "ruff_python_ast", + "ruff_python_literal", + "ruff_python_parser", + "ruff_source_file", + "ruff_text_size", +] + +[[package]] +name = "ruff_python_literal" version = "0.0.0" source = "git+https://github.com/astral-sh/ruff.git?tag=0.13.1#706be0a6e7e09936511198f2ff8982915520d138" dependencies = [ - "aho-corasick", "bitflags 2.9.4", - "compact_str 0.9.0", - "is-macro", "itertools 0.14.0", - "memchr", - "ruff_python_trivia 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.13.1)", - "ruff_source_file 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.13.1)", - "ruff_text_size 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.13.1)", - "rustc-hash", - "thiserror 2.0.16", + "ruff_python_ast", + "unic-ucd-category", ] [[package]] name = "ruff_python_parser" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" +source = "git+https://github.com/astral-sh/ruff.git?tag=0.13.1#706be0a6e7e09936511198f2ff8982915520d138" dependencies = [ "bitflags 2.9.4", "bstr", - "compact_str 0.8.1", + "compact_str", + "get-size2", "memchr", - "ruff_python_ast 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.11.0)", - "ruff_python_trivia 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.11.0)", - "ruff_text_size 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.11.0)", + "ruff_python_ast", + "ruff_python_trivia", + "ruff_text_size", "rustc-hash", "static_assertions", "unicode-ident", @@ -2203,55 +2335,33 @@ dependencies = [ "unicode_names2 1.3.0", ] -[[package]] -name = "ruff_python_trivia" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" -dependencies = [ - "itertools 0.14.0", - "ruff_source_file 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.11.0)", - "ruff_text_size 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.11.0)", - "unicode-ident", -] - [[package]] name = "ruff_python_trivia" version = "0.0.0" source = "git+https://github.com/astral-sh/ruff.git?tag=0.13.1#706be0a6e7e09936511198f2ff8982915520d138" dependencies = [ "itertools 0.14.0", - "ruff_source_file 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.13.1)", - "ruff_text_size 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.13.1)", + "ruff_source_file", + "ruff_text_size", "unicode-ident", ] -[[package]] -name = "ruff_source_file" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" -dependencies = [ - "memchr", - "ruff_text_size 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.11.0)", -] - [[package]] name = "ruff_source_file" version = "0.0.0" source = "git+https://github.com/astral-sh/ruff.git?tag=0.13.1#706be0a6e7e09936511198f2ff8982915520d138" dependencies = [ "memchr", - "ruff_text_size 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.13.1)", + "ruff_text_size", ] -[[package]] -name = "ruff_text_size" -version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" - [[package]] name = "ruff_text_size" version = "0.0.0" source = "git+https://github.com/astral-sh/ruff.git?tag=0.13.1#706be0a6e7e09936511198f2ff8982915520d138" +dependencies = [ + "get-size2", +] [[package]] name = "rustc-hash" @@ -2309,9 +2419,11 @@ dependencies = [ "memchr", "num-complex", "num-traits", - "ruff_python_ast 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.13.1)", + "ruff_python_ast", + "ruff_python_codegen", "ruff_python_parser", - "ruff_text_size 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.11.0)", + "ruff_source_file", + "ruff_text_size", "rustpython-compiler-core", "rustpython-literal", "rustpython-wtf8", @@ -2350,10 +2462,10 @@ dependencies = [ name = "rustpython-compiler" version = "0.4.0" dependencies = [ - "ruff_python_ast 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.13.1)", + "ruff_python_ast", "ruff_python_parser", - "ruff_source_file 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.11.0)", - "ruff_text_size 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.11.0)", + "ruff_source_file", + "ruff_text_size", "rustpython-codegen", "rustpython-compiler-core", "thiserror 2.0.16", @@ -2368,7 +2480,7 @@ dependencies = [ "lz4_flex", "malachite-bigint", "num-complex", - "ruff_source_file 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.11.0)", + "ruff_source_file", "rustpython-wtf8", ] @@ -2570,9 +2682,10 @@ dependencies = [ "parking_lot", "paste", "result-like", - "ruff_python_ast 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.13.1)", + "ruff_python_ast", "ruff_python_parser", - "ruff_text_size 0.0.0 (git+https://github.com/astral-sh/ruff.git?tag=0.11.0)", + "ruff_source_file", + "ruff_text_size", "rustix", "rustpython-codegen", "rustpython-common", diff --git a/Cargo.toml b/Cargo.toml index 4412fb9354a..6008f4da9b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -158,10 +158,11 @@ rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" } rustpython-wtf8 = { path = "wtf8", version = "0.4.0" } rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" } -ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } -ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } -ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } -ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } +ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.13.1" } +ruff_python_codegen = { git = "https://github.com/astral-sh/ruff.git", tag = "0.13.1" } +ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.13.1" } +ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.13.1" } +ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.13.1" } ahash = "0.8.12" ascii = "1.1" diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 27bbe0b64ab..12eccf4d096 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -43,6 +43,7 @@ def test_debug_assignment(self): self.assertEqual(__debug__, prev) setattr(builtins, '__debug__', prev) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised by eval def test_argument_handling(self): # detect duplicate positional and keyword arguments self.assertRaises(SyntaxError, eval, 'lambda a,a:0') diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index cc9f066b141..229f16393fe 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1826,7 +1826,6 @@ def test_gh129093(self): self.assertEqual(f'{f'{1!=2=}'=}', "f'{1!=2=}'='1!=2=True'") self.assertEqual(f'{f'{1 != 2=}'=}', "f'{1 != 2=}'='1 != 2=True'") - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "f-string: newlines are not allowed in format specifiers" does not match "'unexpected EOF while parsing' (, line 2)" def test_newlines_in_format_specifiers(self): cases = [ """f'{1:d\n}'""", diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 9c30054963b..29e2d16203e 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -198,6 +198,7 @@ def _exec_future(self, code): ) return scope + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'a,' != '(a,)' def test_annotations(self): eq = self.assertAnnotationEqual eq('...') @@ -362,6 +363,7 @@ def test_annotations(self): eq('(((a, b)))', '(a, b)') eq("1 + 2 + 3") + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "f'{x=!r}'" != "f'x={x!r}'" def test_fstring_debug_annotations(self): # f-strings with '=' don't round trip very well, so set the expected # result explicitly. @@ -372,6 +374,7 @@ def test_fstring_debug_annotations(self): self.assertAnnotationEqual("f'{x=!a}'", expected="f'x={x!a}'") self.assertAnnotationEqual("f'{x=!s:*^20}'", expected="f'x={x!s:*^20}'") + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '1e309, 1e309j' != '(1e309, 1e309j)' def test_infinity_numbers(self): inf = "1e" + repr(sys.float_info.max_10_exp + 1) infj = f"{inf}j" diff --git a/Lib/test/test_keywordonlyarg.py b/Lib/test/test_keywordonlyarg.py index e41e7c051f6..42bd434478e 100644 --- a/Lib/test/test_keywordonlyarg.py +++ b/Lib/test/test_keywordonlyarg.py @@ -40,6 +40,7 @@ def shouldRaiseSyntaxError(s): compile(s, "", "single") self.assertRaises(SyntaxError, shouldRaiseSyntaxError, codestr) + @unittest.expectedFailureIf(__import__('sys').platform == 'linux', 'TODO: RUSTPYTHON; AssertionError: SyntaxError not raised by shouldRaiseSyntaxError') def testSyntaxErrorForFunctionDefinition(self): self.assertRaisesSyntaxError("def f(p, *):\n pass\n") self.assertRaisesSyntaxError("def f(p1, *, p1=100):\n pass\n") diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 7e46773047e..29dd04995ec 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -259,6 +259,36 @@ Traceback (most recent call last): SyntaxError: invalid syntax +Comprehensions without 'in' keyword: + +>>> [x for x if range(1)] +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> tuple(x for x if range(1)) +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> [x for x() in a] +Traceback (most recent call last): +SyntaxError: cannot assign to function call + +>>> [x for a, b, (c + 1, d()) in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + +>>> [x for a, b, (c + 1, d()) if y] +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> [x for x+1 in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + +>>> [x for x+1, x() in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + Comprehensions creating tuples without parentheses should produce a specialized error message: @@ -322,6 +352,13 @@ Traceback (most recent call last): SyntaxError: invalid syntax +# But prefixes of soft keywords should +# still raise specialized errors + +>>> (mat x) +Traceback (most recent call last): +SyntaxError: invalid syntax. Perhaps you forgot a comma? + From compiler_complex_args(): >>> def f(None=1): @@ -334,7 +371,12 @@ >>> def f(x, y=1, z): ... pass Traceback (most recent call last): -SyntaxError: non-default argument follows default argument +SyntaxError: parameter without a default follows parameter with a default + +>>> def f(x, /, y=1, z): +... pass +Traceback (most recent call last): +SyntaxError: parameter without a default follows parameter with a default >>> def f(x, None): ... pass @@ -555,8 +597,16 @@ Traceback (most recent call last): SyntaxError: expected default value expression -# TODO: RUSTPYTHON NameError: name 'PyCF_TYPE_COMMENTS' is not defined ->>> import ast; ast.parse(''' # doctest: +SKIP +>>> lambda a,d=3,c: None +Traceback (most recent call last): +SyntaxError: parameter without a default follows parameter with a default + +>>> lambda a,/,d=3,c: None +Traceback (most recent call last): +SyntaxError: parameter without a default follows parameter with a default + +>>> # TODO: RUSTPYTHON +>>> import ast; ast.parse(''' # doctest: +SKIP ... def f( ... *, # type: int ... a, # type: int @@ -582,46 +632,31 @@ >>> L = range(10) >>> f(x for x in L) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, 1) # doctest: +SKIP +>>> f(x for x in L, 1) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, y=1) # doctest: +SKIP +>>> f(x for x in L, y=1) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, *[]) # doctest: +SKIP +>>> f(x for x in L, *[]) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, **{}) # doctest: +SKIP +>>> f(x for x in L, **{}) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(L, x for x in L) # doctest: +SKIP +>>> f(L, x for x in L) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, y for y in L) # doctest: +SKIP +>>> f(x for x in L, y for y in L) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L,) # doctest: +SKIP +>>> f(x for x in L,) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized >>> f((x for x in L), 1) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - -# TODO: RUSTPYTHON TypeError: metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases ->>> class C(x for x in L): # doctest: +SKIP +>>> # TODO: RUSTPYTHON +>>> class C(x for x in L): # doctest: +SKIP ... pass Traceback (most recent call last): SyntaxError: invalid syntax @@ -742,8 +777,8 @@ >>> f(x.y=1) Traceback (most recent call last): SyntaxError: expression cannot contain assignment, perhaps you meant "=="? -# TODO: RUSTPYTHON ->>> f((x)=2) # doctest: +SKIP +>>> # TODO: RUSTPYTHON +>>> f((x)=2) # doctest: +SKIP Traceback (most recent call last): SyntaxError: expression cannot contain assignment, perhaps you meant "=="? >>> f(True=1) @@ -758,11 +793,31 @@ >>> f(__debug__=1) Traceback (most recent call last): SyntaxError: cannot assign to __debug__ - -# TODO: RUSTPYTHON NameError: name '__annotations__' is not defined ->>> __debug__: int # doctest: +SKIP +>>> # TODO: RUSTPYTHON +>>> __debug__: int # doctest: +SKIP Traceback (most recent call last): SyntaxError: cannot assign to __debug__ +>>> f(a=) +Traceback (most recent call last): +SyntaxError: expected argument value expression +>>> f(a, b, c=) +Traceback (most recent call last): +SyntaxError: expected argument value expression +>>> f(a, b, c=, d) +Traceback (most recent call last): +SyntaxError: expected argument value expression +>>> f(*args=[0]) +Traceback (most recent call last): +SyntaxError: cannot assign to iterable argument unpacking +>>> f(a, b, *args=[0]) +Traceback (most recent call last): +SyntaxError: cannot assign to iterable argument unpacking +>>> f(**kwargs={'a': 1}) +Traceback (most recent call last): +SyntaxError: cannot assign to keyword argument unpacking +>>> f(a, b, *args, **kwargs={'a': 1}) +Traceback (most recent call last): +SyntaxError: cannot assign to keyword argument unpacking More set_context(): @@ -990,11 +1045,26 @@ Traceback (most recent call last): SyntaxError: expected ':' + >>> def f[T]() + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + >>> class A ... pass Traceback (most recent call last): SyntaxError: expected ':' + >>> class A[T] + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + + >>> class A[T]() + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + >>> class R&D: ... pass Traceback (most recent call last): @@ -1154,6 +1224,22 @@ Traceback (most recent call last): SyntaxError: expected '(' + >>> def f -> int: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f -> int: # type: int + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f[T]: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> def f[T] -> str: + Traceback (most recent call last): + SyntaxError: expected '(' + Parenthesized arguments in function definitions >>> def f(x, (y, z), w): @@ -1432,11 +1518,21 @@ Traceback (most recent call last): IndentationError: expected an indented block after function definition on line 1 + >>> def foo[T](x, /, y, *, z=2): + ... pass + Traceback (most recent call last): + IndentationError: expected an indented block after function definition on line 1 + >>> class Blech(A): ... pass Traceback (most recent call last): IndentationError: expected an indented block after class definition on line 1 + >>> class Blech[T](A): + ... pass + Traceback (most recent call last): + IndentationError: expected an indented block after class definition on line 1 + >>> match something: ... pass Traceback (most recent call last): @@ -1469,14 +1565,16 @@ Check that an multiple exception types with missing parentheses raise a custom exception - >>> try: + >>> # TODO: RUSTPYTHON + >>> try: # doctest: +SKIP ... pass ... except A, B: ... pass Traceback (most recent call last): SyntaxError: multiple exception types must be parenthesized - >>> try: + >>> # TODO: RUSTPYTHON + >>> try: # doctest: +SKIP ... pass ... except A, B, C: ... pass @@ -1591,30 +1689,113 @@ Traceback (most recent call last): SyntaxError: trailing comma not allowed without surrounding parentheses +>>> import a from b +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z from b.y.z +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a from b as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z from b.y.z as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a, b,c from b +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z, b.y.z, c.y.z from b.y.z +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a,b,c from b as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z, b.y.z, c.y.z from b.y.z as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + # Check that we dont raise the "trailing comma" error if there is more # input to the left of the valid part that we parsed. ->>> from t import x,y, and 3 +>>> from t import x,y, and 3 Traceback (most recent call last): SyntaxError: invalid syntax -# TODO: RUSTPYTHON nothing raised. ->>> (): int # doctest: +SKIP +>>> from i import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> from .. import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> (): int Traceback (most recent call last): SyntaxError: only single target (not tuple) can be annotated -# TODO: RUSTPYTHON nothing raised. ->>> []: int # doctest: +SKIP +>>> []: int Traceback (most recent call last): SyntaxError: only single target (not list) can be annotated -# TODO: RUSTPYTHON nothing raised. ->>> (()): int # doctest: +SKIP +>>> (()): int Traceback (most recent call last): SyntaxError: only single target (not tuple) can be annotated -# TODO: RUSTPYTHON nothing raised. ->>> ([]): int # doctest: +SKIP +>>> ([]): int Traceback (most recent call last): SyntaxError: only single target (not list) can be annotated +# 'not' after operators: + +>>> 3 + not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 * not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> + not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> - not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> ~ not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 + - not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 + not -1 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +# Check that we don't introduce misleading errors +>>> not 1 */ 2 +Traceback (most recent call last): +SyntaxError: invalid syntax + +>>> not 1 + +Traceback (most recent call last): +SyntaxError: invalid syntax + +>>> not + 1 + +Traceback (most recent call last): +SyntaxError: invalid syntax + Corner-cases that used to fail to raise the correct error: >>> def f(*, x=lambda __debug__:0): pass @@ -1635,8 +1816,7 @@ Corner-cases that used to crash: - # TODO: RUSTPYTHON nothing raised. - >>> def f(**__debug__): pass # doctest: +SKIP + >>> def f(**__debug__): pass Traceback (most recent call last): SyntaxError: cannot assign to __debug__ @@ -1650,8 +1830,8 @@ Invalid pattern matching constructs: - # TODO: RUSTPYTHON nothing raised. - >>> match ...: # doctest: +SKIP + >>> # TODO: RUSTPYTHON + >>> match ...: # doctest: +SKIP ... case 42 as _: ... ... Traceback (most recent call last): @@ -1751,22 +1931,22 @@ >>> A[*(1:2)] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> A[*(1:2)] = 1 Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> del A[*(1:2)] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression A[*:] and A[:*] >>> A[*:] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> A[:*] Traceback (most recent call last): ... @@ -1777,7 +1957,7 @@ >>> A[*] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression A[**] @@ -1829,10 +2009,246 @@ def f(x: *b) Traceback (most recent call last): ... SyntaxError: invalid syntax + +Invalid bytes literals: + + >>> b"Ā" + Traceback (most recent call last): + ... + b"Ā" + ^^^ + SyntaxError: bytes can only contain ASCII literal characters + + >>> b"абвгде" + Traceback (most recent call last): + ... + b"абвгде" + ^^^^^^^^ + SyntaxError: bytes can only contain ASCII literal characters + + >>> b"abc ъющый" # first 3 letters are ascii + Traceback (most recent call last): + ... + b"abc ъющый" + ^^^^^^^^^^^ + SyntaxError: bytes can only contain ASCII literal characters + +Invalid expressions in type scopes: + + >>> type A[] = int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> class A[]: ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[](): ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[]() + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> async def some[]: # type: int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def f[T: (x:=3)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> def f[T: ((x:= 3), int)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> def f[T = ((x:=3))](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> async def f[T: (x:=3)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> async def f[T: ((x:= 3), int)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> async def f[T = ((x:=3))](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> type A[T: (x:=3)] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> type A[T: ((x:= 3), int)] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> type A[T = ((x:=3))] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> def f[T: (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> def f[T: (int, (yield))](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> def f[T = (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> def f[*Ts = (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVarTuple default + + >>> def f[**P = [(yield), int]](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a ParamSpec default + + >>> type A[T: (yield 3)] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> type A[T: (int, (yield 3))] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> type A[T = (yield 3)] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> type A[T: (await 3)] = int + Traceback (most recent call last): + ... + SyntaxError: await expression cannot be used within a TypeVar bound + + >>> type A[T: (yield from [])] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> class A[T: (yield 3)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> class A[T: (int, (yield 3))]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> class A[T = (yield)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> class A[*Ts = (yield)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVarTuple default + + >>> class A[**P = [(yield), int]]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a ParamSpec default + + >>> # TODO: RUSTPYTHON + >>> type A = (x := 3) # doctest: +SKIP + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a type alias + + >>> type A = (yield 3) + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a type alias + + >>> type A = (await 3) + Traceback (most recent call last): + ... + SyntaxError: await expression cannot be used within a type alias + + >>> type A = (yield from []) + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a type alias + + >>> class A[T]((x := 3)): ... + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within the definition of a generic + + >>> class A[T]((yield 3)): ... + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within the definition of a generic + + >>> class A[T]((await 3)): ... + Traceback (most recent call last): + ... + SyntaxError: await expression cannot be used within the definition of a generic + + >>> class A[T]((yield from [])): ... + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within the definition of a generic + + >>> f(**x, *y) + Traceback (most recent call last): + SyntaxError: iterable argument unpacking follows keyword argument unpacking + + >>> f(**x, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x, *:) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x = 5, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x = 5, *:) + Traceback (most recent call last): + SyntaxError: Invalid star expression """ import re import doctest +import textwrap import unittest from test import support @@ -1869,8 +2285,7 @@ def _check_error(self, code, errtext, else: self.fail("compile() did not raise SyntaxError") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_expression_with_assignment(self): self._check_error( "print(end1 + end2 = ' ')", @@ -1878,16 +2293,14 @@ def test_expression_with_assignment(self): offset=7 ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_curly_brace_after_primary_raises_immediately(self): self._check_error("f{}", "invalid syntax", mode="single") def test_assign_call(self): self._check_error("f() = 1", "assign") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_assign_del(self): self._check_error("del (,)", "invalid syntax") self._check_error("del 1", "cannot delete literal") @@ -1939,9 +2352,6 @@ def error2(): """ self._check_error(source, "parameter and nonlocal", lineno=3) - def test_break_outside_loop(self): - self._check_error("break", "outside loop") - def test_yield_outside_function(self): self._check_error("if 0: yield", "outside function") self._check_error("if 0: yield\nelse: x=1", "outside function") @@ -1970,22 +2380,28 @@ def test_return_outside_function(self): "outside function") def test_break_outside_loop(self): - self._check_error("if 0: break", "outside loop") - self._check_error("if 0: break\nelse: x=1", "outside loop") - self._check_error("if 1: pass\nelse: break", "outside loop") - self._check_error("class C:\n if 0: break", "outside loop") + msg = "outside loop" + self._check_error("break", msg, lineno=1) + self._check_error("if 0: break", msg, lineno=1) + self._check_error("if 0: break\nelse: x=1", msg, lineno=1) + self._check_error("if 1: pass\nelse: break", msg, lineno=2) + self._check_error("class C:\n if 0: break", msg, lineno=2) self._check_error("class C:\n if 1: pass\n else: break", - "outside loop") + msg, lineno=3) + self._check_error("with object() as obj:\n break", + msg, lineno=2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_continue_outside_loop(self): - self._check_error("if 0: continue", "not properly in loop") - self._check_error("if 0: continue\nelse: x=1", "not properly in loop") - self._check_error("if 1: pass\nelse: continue", "not properly in loop") - self._check_error("class C:\n if 0: continue", "not properly in loop") + msg = "not properly in loop" + self._check_error("if 0: continue", msg, lineno=1) + self._check_error("if 0: continue\nelse: x=1", msg, lineno=1) + self._check_error("if 1: pass\nelse: continue", msg, lineno=2) + self._check_error("class C:\n if 0: continue", msg, lineno=2) self._check_error("class C:\n if 1: pass\n else: continue", - "not properly in loop") + msg, lineno=3) + self._check_error("with object() as obj:\n continue", + msg, lineno=2) def test_unexpected_indent(self): self._check_error("foo()\n bar()\n", "unexpected indent", @@ -2000,35 +2416,42 @@ def test_bad_outdent(self): "unindent does not match .* level", subclass=IndentationError) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_kwargs_last(self): self._check_error("int(base=10, '2')", "positional argument follows keyword argument") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_kwargs_last2(self): self._check_error("int(**{'base': 10}, '2')", "positional argument follows " "keyword argument unpacking") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_kwargs_last3(self): self._check_error("int(**{'base': 10}, *['2'])", "iterable argument unpacking follows " "keyword argument unpacking") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_generator_in_function_call(self): self._check_error("foo(x, y for y in range(3) for z in range(2) if z , p)", "Generator expression must be parenthesized", lineno=1, end_lineno=1, offset=11, end_offset=53) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_except_then_except_star(self): + self._check_error("try: pass\nexcept ValueError: pass\nexcept* TypeError: pass", + r"cannot have both 'except' and 'except\*' on the same 'try'", + lineno=3, end_lineno=3, offset=1, end_offset=8) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_except_star_then_except(self): + self._check_error("try: pass\nexcept* ValueError: pass\nexcept TypeError: pass", + r"cannot have both 'except' and 'except\*' on the same 'try'", + lineno=3, end_lineno=3, offset=1, end_offset=7) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_empty_line_after_linecont(self): # See issue-40847 s = r"""\ @@ -2073,6 +2496,25 @@ def test_continuation_bad_indentation(self): self.assertRaises(IndentationError, exec, code) + @support.cpython_only + def test_disallowed_type_param_names(self): + # See gh-128632 + + self._check_error(f"class A[__classdict__]: pass", + f"reserved name '__classdict__' cannot be used for type parameter") + self._check_error(f"def f[__classdict__](): pass", + f"reserved name '__classdict__' cannot be used for type parameter") + self._check_error(f"type T[__classdict__] = tuple[__classdict__]", + f"reserved name '__classdict__' cannot be used for type parameter") + + # These compilations are here to make sure __class__, __classcell__ and __classdictcell__ + # don't break in the future like __classdict__ did in this case. + for name in ('__class__', '__classcell__', '__classdictcell__'): + compile(f""" +class A: + class B[{name}]: pass + """, "", mode="exec") + @support.cpython_only def test_nested_named_except_blocks(self): code = "" @@ -2083,6 +2525,58 @@ def test_nested_named_except_blocks(self): code += f"{' '*4*12}pass" self._check_error(code, "too many statically nested blocks") + @support.cpython_only + def test_with_statement_many_context_managers(self): + # See gh-113297 + + def get_code(n): + code = textwrap.dedent(""" + def bug(): + with ( + a + """) + for i in range(n): + code += f" as a{i}, a\n" + code += "): yield a" + return code + + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block + + for n in range(MAX_MANAGERS): + with self.subTest(f"within range: {n=}"): + compile(get_code(n), "", "exec") + + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): + with self.subTest(f"out of range: {n=}"): + self._check_error(get_code(n), "too many statically nested blocks") + + @support.cpython_only + def test_async_with_statement_many_context_managers(self): + # See gh-116767 + + def get_code(n): + code = [ textwrap.dedent(""" + async def bug(): + async with ( + a + """) ] + for i in range(n): + code.append(f" as a{i}, a\n") + code.append("): yield a") + return "".join(code) + + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block + + for n in range(MAX_MANAGERS): + with self.subTest(f"within range: {n=}"): + compile(get_code(n), "", "exec") + + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): + with self.subTest(f"out of range: {n=}"): + self._check_error(get_code(n), "too many statically nested blocks") + def test_barry_as_flufl_with_syntax_errors(self): # The "barry_as_flufl" rule can produce some "bugs-at-a-distance" if # is reading the wrong token in the presence of syntax errors later @@ -2100,8 +2594,7 @@ def func2(): """ self._check_error(code, "expected ':'") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_line_continuation_error_position(self): self._check_error(r"a = 3 \ 4", "unexpected character after line continuation character", @@ -2113,8 +2606,7 @@ def test_invalid_line_continuation_error_position(self): "unexpected character after line continuation character", lineno=3, offset=4) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_line_continuation_left_recursive(self): # Check bpo-42218: SyntaxErrors following left-recursive rules # (t_primary_raw in this case) need to be tested explicitly @@ -2123,8 +2615,7 @@ def test_invalid_line_continuation_left_recursive(self): self._check_error("A.\u03bc\\\n", "unexpected EOF while parsing") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_error_parenthesis(self): for paren in "([{": self._check_error(paren + "1 + 2", f"\\{paren}' was never closed") @@ -2135,10 +2626,39 @@ def test_error_parenthesis(self): for paren in ")]}": self._check_error(paren + "1 + 2", f"unmatched '\\{paren}'") - # TODO: RUSTPYTHON - @unittest.expectedFailure + # Some more complex examples: + code = """\ +func( + a=["unclosed], # Need a quote in this comment: " + b=2, +) +""" + self._check_error(code, "parenthesis '\\)' does not match opening parenthesis '\\['") + + self._check_error("match y:\n case e(e=v,v,", " was never closed") + + # Examples with dencodings + s = b'# coding=latin\n(aaaaaaaaaaaaaaaaa\naaaaaaaaaaa\xb5' + self._check_error(s, r"'\(' was never closed") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_error_string_literal(self): + + self._check_error("'blech", r"unterminated string literal \(.*\)$") + self._check_error('"blech', r"unterminated string literal \(.*\)$") + self._check_error( + r'"blech\"', r"unterminated string literal \(.*\); perhaps you escaped the end quote" + ) + self._check_error( + r'r"blech\"', r"unterminated string literal \(.*\); perhaps you escaped the end quote" + ) + self._check_error("'''blech", "unterminated triple-quoted string literal") + self._check_error('"""blech', "unterminated triple-quoted string literal") + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invisible_characters(self): self._check_error('print\x17("Hello")', "invalid non-printable character") + self._check_error(b"with(0,,):\n\x01", "invalid non-printable character") def test_match_call_does_not_raise_syntax_error(self): code = """ @@ -2158,8 +2678,7 @@ def case(x): """ compile(code, "", "exec") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_multiline_compiler_error_points_to_the_end(self): self._check_error( "call(\na=1,\na=1\n)", @@ -2198,7 +2717,8 @@ def test_syntax_error_on_deeply_nested_blocks(self): while 20: while 21: while 22: - break + while 23: + break """ self._check_error(source, "too many statically nested blocks") @@ -2207,7 +2727,7 @@ def test_error_on_parser_stack_overflow(self): source = "-" * 100000 + "4" for mode in ["exec", "eval", "single"]: with self.subTest(mode=mode): - with self.assertRaises(MemoryError): + with self.assertRaisesRegex(MemoryError, r"too complex"): compile(source, "", mode) @support.cpython_only diff --git a/compiler/codegen/Cargo.toml b/compiler/codegen/Cargo.toml index ce7e8d74f59..5251ace957e 100644 --- a/compiler/codegen/Cargo.toml +++ b/compiler/codegen/Cargo.toml @@ -12,7 +12,10 @@ license.workspace = true rustpython-compiler-core = { workspace = true } rustpython-literal = {workspace = true } rustpython-wtf8 = { workspace = true } + ruff_python_ast = { workspace = true } +ruff_python_codegen = { workspace = true } +ruff_source_file = { workspace = true } ruff_text_size = { workspace = true } ahash = { workspace = true } diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index c509052a56a..d73ab731e83 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -14,7 +14,6 @@ use crate::{ error::{CodegenError, CodegenErrorType, InternalError, PatternUnreachableReason}, ir::{self, BlockIdx}, symboltable::{self, CompilerScope, SymbolFlags, SymbolScope, SymbolTable}, - unparse::UnparseExpr, }; use itertools::Itertools; use malachite_bigint::BigInt; @@ -24,13 +23,15 @@ use ruff_python_ast::{ Alias, Arguments, BoolOp, CmpOp, Comprehension, ConversionFlag, DebugText, Decorator, DictItem, ExceptHandler, ExceptHandlerExceptHandler, Expr, ExprAttribute, ExprBoolOp, ExprContext, ExprFString, ExprList, ExprName, ExprSlice, ExprStarred, ExprSubscript, ExprTuple, ExprUnaryOp, - FString, FStringElement, FStringElements, FStringFlags, FStringPart, Identifier, Int, Keyword, - MatchCase, ModExpression, ModModule, Operator, Parameters, Pattern, PatternMatchAs, - PatternMatchClass, PatternMatchMapping, PatternMatchOr, PatternMatchSequence, - PatternMatchSingleton, PatternMatchStar, PatternMatchValue, Singleton, Stmt, StmtExpr, - TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, UnaryOp, - WithItem, + FString, FStringFlags, FStringPart, Identifier, Int, InterpolatedStringElement, + InterpolatedStringElements, Keyword, MatchCase, ModExpression, ModModule, Operator, Parameters, + Pattern, PatternMatchAs, PatternMatchClass, PatternMatchMapping, PatternMatchOr, + PatternMatchSequence, PatternMatchSingleton, PatternMatchStar, PatternMatchValue, Singleton, + Stmt, StmtExpr, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, + TypeParams, UnaryOp, WithItem, }; +use ruff_source_file::LineEnding; +use ruff_source_file::PositionEncoding; use ruff_text_size::{Ranged, TextRange}; use rustpython_compiler_core::{ Mode, OneIndexed, SourceFile, SourceLocation, @@ -147,6 +148,14 @@ enum ComprehensionType { Dict, } +fn unparse_expr(expr: &Expr) -> String { + use ruff_python_ast::str::Quote; + use ruff_python_codegen::{Generator, Indentation}; + Generator::new(&Indentation::default(), LineEnding::default()) + .with_preferred_quote(Some(Quote::Single)) + .expr(expr) +} + /// Compile an Mod produced from ruff parser pub fn compile_top( ast: ruff_python_ast::Mod, @@ -240,18 +249,18 @@ fn eprint_location(zelf: &Compiler) { let start = zelf .source_file .to_source_code() - .source_location(zelf.current_source_range.start()); + .source_location(zelf.current_source_range.start(), PositionEncoding::Utf8); let end = zelf .source_file .to_source_code() - .source_location(zelf.current_source_range.end()); + .source_location(zelf.current_source_range.end(), PositionEncoding::Utf8); eprintln!( "LOCATION: {} from {}:{} to {}:{}", zelf.source_file.name(), - start.row, - start.column, - end.row, - end.column + start.line, + start.character_offset, + end.line, + end.character_offset ); } @@ -531,7 +540,7 @@ impl Compiler { let location = self .source_file .to_source_code() - .source_location(range.start()); + .source_location(range.start(), PositionEncoding::Utf8); CodegenError { error, location: Some(location), @@ -631,8 +640,8 @@ impl Compiler { ) -> CompileResult<()> { // Create location let location = SourceLocation { - row: OneIndexed::new(lineno as usize).unwrap_or(OneIndexed::MIN), - column: OneIndexed::new(1).unwrap(), + line: OneIndexed::new(lineno as usize).unwrap_or(OneIndexed::MIN), + character_offset: OneIndexed::new(1).unwrap(), }; // Allocate a new compiler unit @@ -769,8 +778,8 @@ impl Compiler { let _resume_loc = if scope_type == CompilerScope::Module { // Module scope starts with lineno 0 SourceLocation { - row: OneIndexed::MIN, - column: OneIndexed::MIN, + line: OneIndexed::MIN, + character_offset: OneIndexed::MIN, } } else { location @@ -3591,7 +3600,7 @@ impl Compiler { | Expr::NoneLiteral(_) ); let key_repr = if is_literal { - UnparseExpr::new(key, &self.source_file).to_string() + unparse_expr(key) } else if is_attribute { String::new() } else { @@ -4145,9 +4154,7 @@ impl Compiler { fn compile_annotation(&mut self, annotation: &Expr) -> CompileResult<()> { if self.future_annotations { self.emit_load_const(ConstantData::Str { - value: UnparseExpr::new(annotation, &self.source_file) - .to_string() - .into(), + value: unparse_expr(annotation).into(), }); } else { let was_in_annotation = self.in_annotation; @@ -4873,6 +4880,7 @@ impl Compiler { target, value, range: _, + node_index: _, }) => { self.compile_expression(value)?; emit!(self, Instruction::Duplicate); @@ -4931,6 +4939,7 @@ impl Compiler { Expr::IpyEscapeCommand(_) => { panic!("unexpected ipy escape command"); } + Expr::TString(_) => todo!(), } Ok(()) } @@ -5334,7 +5343,7 @@ impl Compiler { let location = self .source_file .to_source_code() - .source_location(range.start()); + .source_location(range.start(), PositionEncoding::Utf8); // TODO: insert source filename self.current_block().instructions.push(ir::InstructionInfo { instr, @@ -5525,25 +5534,30 @@ impl Compiler { target, value, range: _, + node_index: _, }) => Self::contains_await(target) || Self::contains_await(value), - Expr::FString(ExprFString { value, range: _ }) => { + Expr::FString(ExprFString { + value, + range: _, + node_index: _, + }) => { fn expr_element_contains_await bool>( - expr_element: &FStringExpressionElement, + expr_element: &InterpolatedElement, contains_await: F, ) -> bool { contains_await(&expr_element.expression) || expr_element .format_spec .iter() - .flat_map(|spec| spec.elements.expressions()) + .flat_map(|spec| spec.elements.interpolations()) .any(|element| expr_element_contains_await(element, contains_await)) } value.elements().any(|element| match element { - FStringElement::Expression(expr_element) => { + InterpolatedStringElement::Interpolation(expr_element) => { expr_element_contains_await(expr_element, Self::contains_await) } - FStringElement::Literal(_) => false, + InterpolatedStringElement::Literal(_) => false, }) } Expr::StringLiteral(_) @@ -5553,6 +5567,8 @@ impl Compiler { | Expr::NoneLiteral(_) | Expr::EllipsisLiteral(_) | Expr::IpyEscapeCommand(_) => false, + + Expr::TString(_) => todo!(), } } @@ -5602,13 +5618,13 @@ impl Compiler { fn compile_fstring_elements( &mut self, flags: FStringFlags, - fstring_elements: &FStringElements, + fstring_elements: &InterpolatedStringElements, ) -> CompileResult<()> { let mut element_count = 0; for element in fstring_elements { element_count += 1; match element { - FStringElement::Literal(string) => { + InterpolatedStringElement::Literal(string) => { if string.value.contains(char::REPLACEMENT_CHARACTER) { // might have a surrogate literal; should reparse to be sure let source = self.source_file.slice(string.range); @@ -5625,7 +5641,7 @@ impl Compiler { }); } } - FStringElement::Expression(fstring_expr) => { + InterpolatedStringElement::Interpolation(fstring_expr) => { let mut conversion = fstring_expr.conversion; if let Some(DebugText { leading, trailing }) = &fstring_expr.debug_text { @@ -5861,21 +5877,27 @@ mod ruff_tests { range, id: Name::new("x"), ctx: ExprContext::Load, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }); let not_present = &Expr::FString(ExprFString { range, value: FStringValue::single(FString { range, - elements: vec![FStringElement::Expression(FStringExpressionElement { - range, - expression: Box::new(expr_x), - debug_text: None, - conversion: ConversionFlag::None, - format_spec: None, - })] + elements: vec![InterpolatedStringElement::Interpolation( + InterpolatedElement { + range, + expression: Box::new(expr_x), + debug_text: None, + conversion: ConversionFlag::None, + format_spec: None, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + }, + )] .into(), flags, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }); assert!(!Compiler::contains_await(not_present)); @@ -5886,22 +5908,29 @@ mod ruff_tests { range, id: Name::new("x"), ctx: ExprContext::Load, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, })), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }); let present = &Expr::FString(ExprFString { range, value: FStringValue::single(FString { range, - elements: vec![FStringElement::Expression(FStringExpressionElement { - range, - expression: Box::new(expr_await_x), - debug_text: None, - conversion: ConversionFlag::None, - format_spec: None, - })] + elements: vec![InterpolatedStringElement::Interpolation( + InterpolatedElement { + range, + expression: Box::new(expr_await_x), + debug_text: None, + conversion: ConversionFlag::None, + format_spec: None, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + }, + )] .into(), flags, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }); assert!(Compiler::contains_await(present)); @@ -5910,6 +5939,7 @@ mod ruff_tests { range, id: Name::new("x"), ctx: ExprContext::Load, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }); let expr_await_y = Expr::Await(ExprAwait { range, @@ -5917,32 +5947,43 @@ mod ruff_tests { range, id: Name::new("y"), ctx: ExprContext::Load, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, })), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }); let present = &Expr::FString(ExprFString { range, value: FStringValue::single(FString { range, - elements: vec![FStringElement::Expression(FStringExpressionElement { - range, - expression: Box::new(expr_x), - debug_text: None, - conversion: ConversionFlag::None, - format_spec: Some(Box::new(FStringFormatSpec { + elements: vec![InterpolatedStringElement::Interpolation( + InterpolatedElement { range, - elements: vec![FStringElement::Expression(FStringExpressionElement { + expression: Box::new(expr_x), + debug_text: None, + conversion: ConversionFlag::None, + format_spec: Some(Box::new(InterpolatedStringFormatSpec { range, - expression: Box::new(expr_await_y), - debug_text: None, - conversion: ConversionFlag::None, - format_spec: None, - })] - .into(), - })), - })] + elements: vec![InterpolatedStringElement::Interpolation( + InterpolatedElement { + range, + expression: Box::new(expr_await_y), + debug_text: None, + conversion: ConversionFlag::None, + format_spec: None, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + }, + )] + .into(), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + })), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + }, + )] .into(), flags, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }); assert!(Compiler::contains_await(present)); } diff --git a/compiler/codegen/src/ir.rs b/compiler/codegen/src/ir.rs index 31c89260913..50657ebd978 100644 --- a/compiler/codegen/src/ir.rs +++ b/compiler/codegen/src/ir.rs @@ -182,7 +182,7 @@ impl CodeInfo { *arg = new_arg; } let (extras, lo_arg) = arg.split(); - locations.extend(std::iter::repeat_n(info.location.clone(), arg.instr_size())); + locations.extend(std::iter::repeat_n(info.location, arg.instr_size())); instructions.extend( extras .map(|byte| CodeUnit::new(Instruction::ExtendedArg, byte)) @@ -422,8 +422,8 @@ fn generate_linetable(locations: &[SourceLocation], first_line: i32) -> Box<[u8] // Get line and column information // SourceLocation always has row and column (both are OneIndexed) - let line = loc.row.get() as i32; - let col = (loc.column.get() as i32) - 1; // Convert 1-based to 0-based + let line = loc.line.get() as i32; + let col = (loc.character_offset.get() as i32) - 1; // Convert 1-based to 0-based let line_delta = line - prev_line; diff --git a/compiler/codegen/src/lib.rs b/compiler/codegen/src/lib.rs index 9b444de994d..e7944079988 100644 --- a/compiler/codegen/src/lib.rs +++ b/compiler/codegen/src/lib.rs @@ -13,7 +13,6 @@ pub mod error; pub mod ir; mod string_parser; pub mod symboltable; -mod unparse; pub use compile::CompileOpts; use ruff_python_ast::Expr; @@ -56,6 +55,7 @@ impl ToPythonName for Expr { Self::Starred { .. } => "starred", Self::Slice { .. } => "slice", Self::FString { .. } => "f-string expression", + Self::TString { .. } => "t-string expression", Self::Name { .. } => "name", Self::Lambda { .. } => "lambda", Self::If { .. } => "conditional expression", diff --git a/compiler/codegen/src/symboltable.rs b/compiler/codegen/src/symboltable.rs index 16e88ad6908..229332010d7 100644 --- a/compiler/codegen/src/symboltable.rs +++ b/compiler/codegen/src/symboltable.rs @@ -18,6 +18,7 @@ use ruff_python_ast::{ PatternMatchMapping, PatternMatchOr, PatternMatchSequence, PatternMatchStar, PatternMatchValue, Stmt, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, }; +use ruff_source_file::PositionEncoding; use ruff_text_size::{Ranged, TextRange}; use rustpython_compiler_core::{SourceFile, SourceLocation}; use std::{borrow::Cow, fmt}; @@ -793,6 +794,7 @@ impl SymbolTableBuilder { decorator_list, type_params, range, + node_index: _, }) => { if let Some(type_params) = type_params { self.enter_type_param_block( @@ -910,6 +912,7 @@ impl SymbolTableBuilder { value, simple, range, + node_index: _, }) => { // https://github.com/python/cpython/blob/main/Python/symtable.c#L1233 match &**target { @@ -1046,7 +1049,7 @@ impl SymbolTableBuilder { location: Some( self.source_file .to_source_code() - .source_location(expression.range().start()), + .source_location(expression.range().start(), PositionEncoding::Utf8), ), }); } @@ -1089,7 +1092,12 @@ impl SymbolTableBuilder { }) => { self.scan_expression(value, ExpressionContext::Load)?; } - Expr::Dict(ExprDict { items, range: _ }) => { + Expr::Dict(ExprDict { + items, + range: _, + + node_index: _, + }) => { for item in items { if let Some(key) = &item.key { self.scan_expression(key, context)?; @@ -1097,15 +1105,27 @@ impl SymbolTableBuilder { self.scan_expression(&item.value, context)?; } } - Expr::Await(ExprAwait { value, range: _ }) => { + Expr::Await(ExprAwait { + value, + range: _, + node_index: _, + }) => { self.scan_expression(value, context)?; } - Expr::Yield(ExprYield { value, range: _ }) => { + Expr::Yield(ExprYield { + value, + range: _, + node_index: _, + }) => { if let Some(expression) = value { self.scan_expression(expression, context)?; } } - Expr::YieldFrom(ExprYieldFrom { value, range: _ }) => { + Expr::YieldFrom(ExprYieldFrom { + value, + range: _, + node_index: _, + }) => { self.scan_expression(value, context)?; } Expr::UnaryOp(ExprUnaryOp { @@ -1128,6 +1148,7 @@ impl SymbolTableBuilder { upper, step, range: _, + node_index: _, }) => { if let Some(lower) = lower { self.scan_expression(lower, context)?; @@ -1151,6 +1172,7 @@ impl SymbolTableBuilder { elt, generators, range, + node_index: _, }) => { self.scan_comprehension("genexpr", elt, None, generators, *range)?; } @@ -1158,6 +1180,7 @@ impl SymbolTableBuilder { elt, generators, range, + node_index: _, }) => { self.scan_comprehension("genexpr", elt, None, generators, *range)?; } @@ -1166,6 +1189,7 @@ impl SymbolTableBuilder { value, generators, range, + node_index: _, }) => { self.scan_comprehension("genexpr", key, Some(value), generators, *range)?; } @@ -1173,6 +1197,7 @@ impl SymbolTableBuilder { func, arguments, range: _, + node_index: _, }) => { match context { ExpressionContext::IterDefinitionExp => { @@ -1219,6 +1244,7 @@ impl SymbolTableBuilder { body, parameters, range: _, + node_index: _, }) => { if let Some(parameters) = parameters { self.enter_scope_with_parameters( @@ -1244,10 +1270,10 @@ impl SymbolTableBuilder { self.leave_scope(); } Expr::FString(ExprFString { value, .. }) => { - for expr in value.elements().filter_map(|x| x.as_expression()) { + for expr in value.elements().filter_map(|x| x.as_interpolation()) { self.scan_expression(&expr.expression, ExpressionContext::Load)?; if let Some(format_spec) = &expr.format_spec { - for element in format_spec.elements.expressions() { + for element in format_spec.elements.interpolations() { self.scan_expression(&element.expression, ExpressionContext::Load)? } } @@ -1266,6 +1292,7 @@ impl SymbolTableBuilder { body, orelse, range: _, + node_index: _, }) => { self.scan_expression(test, ExpressionContext::Load)?; self.scan_expression(body, ExpressionContext::Load)?; @@ -1276,13 +1303,14 @@ impl SymbolTableBuilder { target, value, range, + node_index: _, }) => { // named expressions are not allowed in the definition of // comprehension iterator definitions if let ExpressionContext::IterDefinitionExp = context { return Err(SymbolTableError { error: "assignment expression cannot be used in a comprehension iterable expression".to_string(), - location: Some(self.source_file.to_source_code().source_location(target.range().start())), + location: Some(self.source_file.to_source_code().source_location( target.range().start(), PositionEncoding::Utf8)), }); } @@ -1311,6 +1339,7 @@ impl SymbolTableBuilder { self.scan_expression(target, ExpressionContext::Store)?; } } + Expr::TString(_) => todo!(), } Ok(()) } @@ -1393,6 +1422,7 @@ impl SymbolTableBuilder { bound, range: type_var_range, default, + node_index: _, }) => { self.register_name(name.as_str(), SymbolUsage::TypeParam, *type_var_range)?; @@ -1416,6 +1446,7 @@ impl SymbolTableBuilder { name, range: param_spec_range, default, + node_index: _, }) => { self.register_name(name, SymbolUsage::TypeParam, *param_spec_range)?; @@ -1429,6 +1460,7 @@ impl SymbolTableBuilder { name, range: type_var_tuple_range, default, + node_index: _, }) => { self.register_name(name, SymbolUsage::TypeParam, *type_var_tuple_range)?; @@ -1565,7 +1597,7 @@ impl SymbolTableBuilder { let location = self .source_file .to_source_code() - .source_location(range.start()); + .source_location(range.start(), PositionEncoding::Utf8); let location = Some(location); let scope_depth = self.tables.len(); let table = self.tables.last_mut().unwrap(); diff --git a/compiler/codegen/src/unparse.rs b/compiler/codegen/src/unparse.rs deleted file mode 100644 index f2fb86ed4ef..00000000000 --- a/compiler/codegen/src/unparse.rs +++ /dev/null @@ -1,633 +0,0 @@ -use ruff_python_ast::{ - self as ruff, Arguments, BoolOp, Comprehension, ConversionFlag, Expr, Identifier, Operator, - Parameter, ParameterWithDefault, Parameters, -}; -use ruff_text_size::Ranged; -use rustpython_compiler_core::SourceFile; -use rustpython_literal::escape::{AsciiEscape, UnicodeEscape}; -use std::fmt::{self, Display as _}; - -mod precedence { - macro_rules! precedence { - ($($op:ident,)*) => { - precedence!(@0, $($op,)*); - }; - (@$i:expr, $op1:ident, $($op:ident,)*) => { - pub const $op1: u8 = $i; - precedence!(@$i + 1, $($op,)*); - }; - (@$i:expr,) => {}; - } - precedence!( - TUPLE, TEST, OR, AND, NOT, CMP, // "EXPR" = - BOR, BXOR, BAND, SHIFT, ARITH, TERM, FACTOR, POWER, AWAIT, ATOM, - ); - pub const EXPR: u8 = BOR; -} - -struct Unparser<'a, 'b, 'c> { - f: &'b mut fmt::Formatter<'a>, - source: &'c SourceFile, -} - -impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { - const fn new(f: &'b mut fmt::Formatter<'a>, source: &'c SourceFile) -> Self { - Self { f, source } - } - - fn p(&mut self, s: &str) -> fmt::Result { - self.f.write_str(s) - } - - fn p_id(&mut self, s: &Identifier) -> fmt::Result { - self.f.write_str(s.as_str()) - } - - fn p_if(&mut self, cond: bool, s: &str) -> fmt::Result { - if cond { - self.f.write_str(s)?; - } - Ok(()) - } - - fn p_delim(&mut self, first: &mut bool, s: &str) -> fmt::Result { - self.p_if(!std::mem::take(first), s) - } - - fn write_fmt(&mut self, f: fmt::Arguments<'_>) -> fmt::Result { - self.f.write_fmt(f) - } - - fn unparse_expr(&mut self, ast: &Expr, level: u8) -> fmt::Result { - macro_rules! op_prec { - ($op_ty:ident, $x:expr, $enu:path, $($var:ident($op:literal, $prec:ident)),*$(,)?) => { - match $x { - $(<$enu>::$var => (op_prec!(@space $op_ty, $op), precedence::$prec),)* - } - }; - (@space bin, $op:literal) => { - concat!(" ", $op, " ") - }; - (@space un, $op:literal) => { - $op - }; - } - macro_rules! group_if { - ($lvl:expr, $body:block) => {{ - let group = level > $lvl; - self.p_if(group, "(")?; - let ret = $body; - self.p_if(group, ")")?; - ret - }}; - } - match &ast { - Expr::BoolOp(ruff::ExprBoolOp { - op, - values, - range: _range, - }) => { - let (op, prec) = op_prec!(bin, op, BoolOp, And("and", AND), Or("or", OR)); - group_if!(prec, { - let mut first = true; - for val in values { - self.p_delim(&mut first, op)?; - self.unparse_expr(val, prec + 1)?; - } - }) - } - Expr::Named(ruff::ExprNamed { - target, - value, - range: _range, - }) => { - group_if!(precedence::TUPLE, { - self.unparse_expr(target, precedence::ATOM)?; - self.p(" := ")?; - self.unparse_expr(value, precedence::ATOM)?; - }) - } - Expr::BinOp(ruff::ExprBinOp { - left, - op, - right, - range: _range, - }) => { - let right_associative = matches!(op, Operator::Pow); - let (op, prec) = op_prec!( - bin, - op, - Operator, - Add("+", ARITH), - Sub("-", ARITH), - Mult("*", TERM), - MatMult("@", TERM), - Div("/", TERM), - Mod("%", TERM), - Pow("**", POWER), - LShift("<<", SHIFT), - RShift(">>", SHIFT), - BitOr("|", BOR), - BitXor("^", BXOR), - BitAnd("&", BAND), - FloorDiv("//", TERM), - ); - group_if!(prec, { - self.unparse_expr(left, prec + right_associative as u8)?; - self.p(op)?; - self.unparse_expr(right, prec + !right_associative as u8)?; - }) - } - Expr::UnaryOp(ruff::ExprUnaryOp { - op, - operand, - range: _range, - }) => { - let (op, prec) = op_prec!( - un, - op, - ruff::UnaryOp, - Invert("~", FACTOR), - Not("not ", NOT), - UAdd("+", FACTOR), - USub("-", FACTOR) - ); - group_if!(prec, { - self.p(op)?; - self.unparse_expr(operand, prec)?; - }) - } - Expr::Lambda(ruff::ExprLambda { - parameters, - body, - range: _range, - }) => { - group_if!(precedence::TEST, { - if let Some(parameters) = parameters { - self.p("lambda ")?; - self.unparse_arguments(parameters)?; - } else { - self.p("lambda")?; - } - write!(self, ": {}", UnparseExpr::new(body, self.source))?; - }) - } - Expr::If(ruff::ExprIf { - test, - body, - orelse, - range: _range, - }) => { - group_if!(precedence::TEST, { - self.unparse_expr(body, precedence::TEST + 1)?; - self.p(" if ")?; - self.unparse_expr(test, precedence::TEST + 1)?; - self.p(" else ")?; - self.unparse_expr(orelse, precedence::TEST)?; - }) - } - Expr::Dict(ruff::ExprDict { - items, - range: _range, - }) => { - self.p("{")?; - let mut first = true; - for item in items { - self.p_delim(&mut first, ", ")?; - if let Some(k) = &item.key { - write!(self, "{}: ", UnparseExpr::new(k, self.source))?; - } else { - self.p("**")?; - } - self.unparse_expr(&item.value, level)?; - } - self.p("}")?; - } - Expr::Set(ruff::ExprSet { - elts, - range: _range, - }) => { - self.p("{")?; - let mut first = true; - for v in elts { - self.p_delim(&mut first, ", ")?; - self.unparse_expr(v, precedence::TEST)?; - } - self.p("}")?; - } - Expr::ListComp(ruff::ExprListComp { - elt, - generators, - range: _range, - }) => { - self.p("[")?; - self.unparse_expr(elt, precedence::TEST)?; - self.unparse_comp(generators)?; - self.p("]")?; - } - Expr::SetComp(ruff::ExprSetComp { - elt, - generators, - range: _range, - }) => { - self.p("{")?; - self.unparse_expr(elt, precedence::TEST)?; - self.unparse_comp(generators)?; - self.p("}")?; - } - Expr::DictComp(ruff::ExprDictComp { - key, - value, - generators, - range: _range, - }) => { - self.p("{")?; - self.unparse_expr(key, precedence::TEST)?; - self.p(": ")?; - self.unparse_expr(value, precedence::TEST)?; - self.unparse_comp(generators)?; - self.p("}")?; - } - Expr::Generator(ruff::ExprGenerator { - parenthesized: _, - elt, - generators, - range: _range, - }) => { - self.p("(")?; - self.unparse_expr(elt, precedence::TEST)?; - self.unparse_comp(generators)?; - self.p(")")?; - } - Expr::Await(ruff::ExprAwait { - value, - range: _range, - }) => { - group_if!(precedence::AWAIT, { - self.p("await ")?; - self.unparse_expr(value, precedence::ATOM)?; - }) - } - Expr::Yield(ruff::ExprYield { - value, - range: _range, - }) => { - if let Some(value) = value { - write!(self, "(yield {})", UnparseExpr::new(value, self.source))?; - } else { - self.p("(yield)")?; - } - } - Expr::YieldFrom(ruff::ExprYieldFrom { - value, - range: _range, - }) => { - write!( - self, - "(yield from {})", - UnparseExpr::new(value, self.source) - )?; - } - Expr::Compare(ruff::ExprCompare { - left, - ops, - comparators, - range: _range, - }) => { - group_if!(precedence::CMP, { - let new_lvl = precedence::CMP + 1; - self.unparse_expr(left, new_lvl)?; - for (op, cmp) in ops.iter().zip(comparators) { - self.p(" ")?; - self.p(op.as_str())?; - self.p(" ")?; - self.unparse_expr(cmp, new_lvl)?; - } - }) - } - Expr::Call(ruff::ExprCall { - func, - arguments: Arguments { args, keywords, .. }, - range: _range, - }) => { - self.unparse_expr(func, precedence::ATOM)?; - self.p("(")?; - if let ( - [ - Expr::Generator(ruff::ExprGenerator { - elt, - generators, - range: _range, - .. - }), - ], - [], - ) = (&**args, &**keywords) - { - // make sure a single genexpr doesn't get double parens - self.unparse_expr(elt, precedence::TEST)?; - self.unparse_comp(generators)?; - } else { - let mut first = true; - for arg in args { - self.p_delim(&mut first, ", ")?; - self.unparse_expr(arg, precedence::TEST)?; - } - for kw in keywords { - self.p_delim(&mut first, ", ")?; - if let Some(arg) = &kw.arg { - self.p_id(arg)?; - self.p("=")?; - } else { - self.p("**")?; - } - self.unparse_expr(&kw.value, precedence::TEST)?; - } - } - self.p(")")?; - } - Expr::FString(ruff::ExprFString { value, .. }) => self.unparse_fstring(value)?, - Expr::StringLiteral(ruff::ExprStringLiteral { value, .. }) => { - if value.is_unicode() { - self.p("u")? - } - UnicodeEscape::new_repr(value.to_str().as_ref()) - .str_repr() - .fmt(self.f)? - } - Expr::BytesLiteral(ruff::ExprBytesLiteral { value, .. }) => { - AsciiEscape::new_repr(&value.bytes().collect::>()) - .bytes_repr() - .fmt(self.f)? - } - Expr::NumberLiteral(ruff::ExprNumberLiteral { value, .. }) => { - const { assert!(f64::MAX_10_EXP == 308) }; - let inf_str = "1e309"; - match value { - ruff::Number::Int(int) => int.fmt(self.f)?, - &ruff::Number::Float(fp) => { - if fp.is_infinite() { - self.p(inf_str)? - } else { - self.p(&rustpython_literal::float::to_string(fp))? - } - } - &ruff::Number::Complex { real, imag } => self - .p(&rustpython_literal::complex::to_string(real, imag) - .replace("inf", inf_str))?, - } - } - Expr::BooleanLiteral(ruff::ExprBooleanLiteral { value, .. }) => { - self.p(if *value { "True" } else { "False" })? - } - Expr::NoneLiteral(ruff::ExprNoneLiteral { .. }) => self.p("None")?, - Expr::EllipsisLiteral(ruff::ExprEllipsisLiteral { .. }) => self.p("...")?, - Expr::Attribute(ruff::ExprAttribute { value, attr, .. }) => { - self.unparse_expr(value, precedence::ATOM)?; - let period = if let Expr::NumberLiteral(ruff::ExprNumberLiteral { - value: ruff::Number::Int(_), - .. - }) = value.as_ref() - { - " ." - } else { - "." - }; - self.p(period)?; - self.p_id(attr)?; - } - Expr::Subscript(ruff::ExprSubscript { value, slice, .. }) => { - self.unparse_expr(value, precedence::ATOM)?; - let lvl = precedence::TUPLE; - self.p("[")?; - self.unparse_expr(slice, lvl)?; - self.p("]")?; - } - Expr::Starred(ruff::ExprStarred { value, .. }) => { - self.p("*")?; - self.unparse_expr(value, precedence::EXPR)?; - } - Expr::Name(ruff::ExprName { id, .. }) => self.p(id.as_str())?, - Expr::List(ruff::ExprList { elts, .. }) => { - self.p("[")?; - let mut first = true; - for elt in elts { - self.p_delim(&mut first, ", ")?; - self.unparse_expr(elt, precedence::TEST)?; - } - self.p("]")?; - } - Expr::Tuple(ruff::ExprTuple { elts, .. }) => { - if elts.is_empty() { - self.p("()")?; - } else { - group_if!(precedence::TUPLE, { - let mut first = true; - for elt in elts { - self.p_delim(&mut first, ", ")?; - self.unparse_expr(elt, precedence::TEST)?; - } - self.p_if(elts.len() == 1, ",")?; - }) - } - } - Expr::Slice(ruff::ExprSlice { - lower, - upper, - step, - range: _range, - }) => { - if let Some(lower) = lower { - self.unparse_expr(lower, precedence::TEST)?; - } - self.p(":")?; - if let Some(upper) = upper { - self.unparse_expr(upper, precedence::TEST)?; - } - if let Some(step) = step { - self.p(":")?; - self.unparse_expr(step, precedence::TEST)?; - } - } - Expr::IpyEscapeCommand(_) => {} - } - Ok(()) - } - - fn unparse_arguments(&mut self, args: &Parameters) -> fmt::Result { - let mut first = true; - for (i, arg) in args.posonlyargs.iter().chain(&args.args).enumerate() { - self.p_delim(&mut first, ", ")?; - self.unparse_function_arg(arg)?; - self.p_if(i + 1 == args.posonlyargs.len(), ", /")?; - } - if args.vararg.is_some() || !args.kwonlyargs.is_empty() { - self.p_delim(&mut first, ", ")?; - self.p("*")?; - } - if let Some(vararg) = &args.vararg { - self.unparse_arg(vararg)?; - } - for kwarg in &args.kwonlyargs { - self.p_delim(&mut first, ", ")?; - self.unparse_function_arg(kwarg)?; - } - if let Some(kwarg) = &args.kwarg { - self.p_delim(&mut first, ", ")?; - self.p("**")?; - self.unparse_arg(kwarg)?; - } - Ok(()) - } - fn unparse_function_arg(&mut self, arg: &ParameterWithDefault) -> fmt::Result { - self.unparse_arg(&arg.parameter)?; - if let Some(default) = &arg.default { - write!(self, "={}", UnparseExpr::new(default, self.source))?; - } - Ok(()) - } - - fn unparse_arg(&mut self, arg: &Parameter) -> fmt::Result { - self.p_id(&arg.name)?; - if let Some(ann) = &arg.annotation { - write!(self, ": {}", UnparseExpr::new(ann, self.source))?; - } - Ok(()) - } - - fn unparse_comp(&mut self, generators: &[Comprehension]) -> fmt::Result { - for comp in generators { - self.p(if comp.is_async { - " async for " - } else { - " for " - })?; - self.unparse_expr(&comp.target, precedence::TUPLE)?; - self.p(" in ")?; - self.unparse_expr(&comp.iter, precedence::TEST + 1)?; - for cond in &comp.ifs { - self.p(" if ")?; - self.unparse_expr(cond, precedence::TEST + 1)?; - } - } - Ok(()) - } - - fn unparse_fstring_body(&mut self, elements: &[ruff::FStringElement]) -> fmt::Result { - for elem in elements { - self.unparse_fstring_elem(elem)?; - } - Ok(()) - } - - fn unparse_formatted( - &mut self, - val: &Expr, - debug_text: Option<&ruff::DebugText>, - conversion: ConversionFlag, - spec: Option<&ruff::FStringFormatSpec>, - ) -> fmt::Result { - let buffered = to_string_fmt(|f| { - Unparser::new(f, self.source).unparse_expr(val, precedence::TEST + 1) - }); - if let Some(ruff::DebugText { leading, trailing }) = debug_text { - self.p(leading)?; - self.p(self.source.slice(val.range()))?; - self.p(trailing)?; - } - let brace = if buffered.starts_with('{') { - // put a space to avoid escaping the bracket - "{ " - } else { - "{" - }; - self.p(brace)?; - self.p(&buffered)?; - drop(buffered); - - if conversion != ConversionFlag::None { - self.p("!")?; - let buf = &[conversion as u8]; - let c = std::str::from_utf8(buf).unwrap(); - self.p(c)?; - } - - if let Some(spec) = spec { - self.p(":")?; - self.unparse_fstring_body(&spec.elements)?; - } - - self.p("}")?; - - Ok(()) - } - - fn unparse_fstring_elem(&mut self, elem: &ruff::FStringElement) -> fmt::Result { - match elem { - ruff::FStringElement::Expression(ruff::FStringExpressionElement { - expression, - debug_text, - conversion, - format_spec, - .. - }) => self.unparse_formatted( - expression, - debug_text.as_ref(), - *conversion, - format_spec.as_deref(), - ), - ruff::FStringElement::Literal(ruff::FStringLiteralElement { value, .. }) => { - self.unparse_fstring_str(value) - } - } - } - - fn unparse_fstring_str(&mut self, s: &str) -> fmt::Result { - let s = s.replace('{', "{{").replace('}', "}}"); - self.p(&s) - } - - fn unparse_fstring(&mut self, value: &ruff::FStringValue) -> fmt::Result { - self.p("f")?; - let body = to_string_fmt(|f| { - value.iter().try_for_each(|part| match part { - ruff::FStringPart::Literal(lit) => f.write_str(lit), - ruff::FStringPart::FString(ruff::FString { elements, .. }) => { - Unparser::new(f, self.source).unparse_fstring_body(elements) - } - }) - }); - // .unparse_fstring_body(elements)); - UnicodeEscape::new_repr(body.as_str().as_ref()) - .str_repr() - .write(self.f) - } -} - -pub struct UnparseExpr<'a> { - expr: &'a Expr, - source: &'a SourceFile, -} - -impl<'a> UnparseExpr<'a> { - pub const fn new(expr: &'a Expr, source: &'a SourceFile) -> Self { - Self { expr, source } - } -} - -impl fmt::Display for UnparseExpr<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - Unparser::new(f, self.source).unparse_expr(self.expr, precedence::TEST) - } -} - -fn to_string_fmt(f: impl FnOnce(&mut fmt::Formatter<'_>) -> fmt::Result) -> String { - use std::cell::Cell; - struct Fmt(Cell>); - impl) -> fmt::Result> fmt::Display for Fmt { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.take().unwrap()(f) - } - } - Fmt(Cell::new(Some(f))).to_string() -} diff --git a/compiler/core/src/bytecode.rs b/compiler/core/src/bytecode.rs index c2ce4e52c07..a6368bbca15 100644 --- a/compiler/core/src/bytecode.rs +++ b/compiler/core/src/bytecode.rs @@ -1190,14 +1190,14 @@ impl CodeObject { level: usize, ) -> fmt::Result { let label_targets = self.label_targets(); - let line_digits = (3).max(self.locations.last().unwrap().row.to_string().len()); + let line_digits = (3).max(self.locations.last().unwrap().line.to_string().len()); let offset_digits = (4).max(self.instructions.len().to_string().len()); let mut last_line = OneIndexed::MAX; let mut arg_state = OpArgState::default(); for (offset, &instruction) in self.instructions.iter().enumerate() { let (instruction, arg) = arg_state.get(instruction); // optional line number - let line = self.locations[offset].row; + let line = self.locations[offset].line; if line != last_line { if last_line != OneIndexed::MAX { writeln!(f)?; diff --git a/compiler/core/src/marshal.rs b/compiler/core/src/marshal.rs index b8044a1ab95..5e16e59102a 100644 --- a/compiler/core/src/marshal.rs +++ b/compiler/core/src/marshal.rs @@ -198,8 +198,8 @@ pub fn deserialize_code( let locations = (0..len) .map(|_| { Ok(SourceLocation { - row: OneIndexed::new(rdr.read_u32()? as _).ok_or(MarshalError::InvalidLocation)?, - column: OneIndexed::from_zero_indexed(rdr.read_u32()? as _), + line: OneIndexed::new(rdr.read_u32()? as _).ok_or(MarshalError::InvalidLocation)?, + character_offset: OneIndexed::from_zero_indexed(rdr.read_u32()? as _), }) }) .collect::>>()?; @@ -656,8 +656,8 @@ pub fn serialize_code(buf: &mut W, code: &CodeObject) write_len(buf, code.locations.len()); for loc in &*code.locations { - buf.write_u32(loc.row.get() as _); - buf.write_u32(loc.column.to_zero_indexed() as _); + buf.write_u32(loc.line.get() as _); + buf.write_u32(loc.character_offset.to_zero_indexed() as _); } buf.write_u16(code.flags.bits()); diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs index 111b9aec2e7..6c46892150e 100644 --- a/compiler/src/lib.rs +++ b/compiler/src/lib.rs @@ -1,4 +1,4 @@ -use ruff_source_file::{SourceFile, SourceFileBuilder, SourceLocation}; +use ruff_source_file::{PositionEncoding, SourceFile, SourceFileBuilder, SourceLocation}; use rustpython_codegen::{compile, symboltable}; pub use rustpython_codegen::compile::CompileOpts; @@ -46,7 +46,7 @@ impl CompileError { pub fn from_ruff_parse_error(error: parser::ParseError, source_file: &SourceFile) -> Self { let location = source_file .to_source_code() - .source_location(error.location.start()); + .source_location(error.location.start(), PositionEncoding::Utf8); Self::Parse(ParseError { error: error.error, raw_location: error.location, @@ -57,8 +57,8 @@ impl CompileError { pub fn location(&self) -> Option { match self { - Self::Codegen(codegen_error) => codegen_error.location.clone(), - Self::Parse(parse_error) => Some(parse_error.location.clone()), + Self::Codegen(codegen_error) => codegen_error.location, + Self::Parse(parse_error) => Some(parse_error.location), } } @@ -66,14 +66,14 @@ impl CompileError { match self { Self::Codegen(codegen_error) => { if let Some(location) = &codegen_error.location { - (location.row.get(), location.column.get()) + (location.line.get(), location.character_offset.get()) } else { (0, 0) } } Self::Parse(parse_error) => ( - parse_error.location.row.get(), - parse_error.location.column.get(), + parse_error.location.line.get(), + parse_error.location.character_offset.get(), ), } } diff --git a/src/shell.rs b/src/shell.rs index e38b0a50def..15918349665 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -1,7 +1,7 @@ mod helper; use rustpython_compiler::{ - CompileError, ParseError, parser::FStringErrorType, parser::LexicalErrorType, + CompileError, ParseError, parser::InterpolatedStringErrorType, parser::LexicalErrorType, parser::ParseErrorType, }; use rustpython_vm::{ @@ -52,7 +52,7 @@ fn shell_exec( Err(CompileError::Parse(ParseError { error: ParseErrorType::Lexical(LexicalErrorType::FStringError( - FStringErrorType::UnterminatedTripleQuotedString, + InterpolatedStringErrorType::UnterminatedTripleQuotedString, )), .. })) => ShellExecResult::ContinueLine, diff --git a/stdlib/src/faulthandler.rs b/stdlib/src/faulthandler.rs index f358129c873..5c9196ad33f 100644 --- a/stdlib/src/faulthandler.rs +++ b/stdlib/src/faulthandler.rs @@ -10,7 +10,7 @@ mod decl { stderr, " File \"{}\", line {} in {}", frame.code.source_path, - frame.current_location().row, + frame.current_location().line, frame.code.obj_name ) } diff --git a/vm/Cargo.toml b/vm/Cargo.toml index ffd58361789..5f3c8a66920 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -20,7 +20,7 @@ freeze-stdlib = ["encodings"] jit = ["rustpython-jit"] threading = ["rustpython-common/threading"] compiler = ["parser", "codegen", "rustpython-compiler"] -ast = ["ruff_python_ast", "ruff_text_size"] +ast = ["ruff_python_ast", "ruff_text_size", "ruff_source_file"] codegen = ["rustpython-codegen", "ast"] parser = ["ast"] serde = ["dep:serde"] @@ -35,6 +35,7 @@ rustpython-jit = { workspace = true, optional = true } ruff_python_ast = { workspace = true, optional = true } ruff_python_parser = { workspace = true } +ruff_source_file = { workspace = true, optional = true } ruff_text_size = { workspace = true, optional = true } rustpython-compiler-core = { workspace = true } rustpython-literal = { workspace = true } diff --git a/vm/src/builtins/frame.rs b/vm/src/builtins/frame.rs index 65ac3e798d7..17dc88ac042 100644 --- a/vm/src/builtins/frame.rs +++ b/vm/src/builtins/frame.rs @@ -60,7 +60,7 @@ impl Frame { #[pygetset] pub fn f_lineno(&self) -> usize { - self.current_location().row.get() + self.current_location().line.get() } #[pygetset] diff --git a/vm/src/frame.rs b/vm/src/frame.rs index ba9abffc6e7..fa5860244f1 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -171,7 +171,7 @@ impl Frame { } pub fn current_location(&self) -> SourceLocation { - self.code.locations[self.lasti() as usize - 1].clone() + self.code.locations[self.lasti() as usize - 1] } pub fn lasti(&self) -> u32 { @@ -388,11 +388,15 @@ impl ExecutingFrame<'_> { // 2. Add new entry with current execution position (filename, lineno, code_object) to traceback. // 3. Unwind block stack till appropriate handler is found. - let loc = frame.code.locations[idx].clone(); + let loc = frame.code.locations[idx]; let next = exception.__traceback__(); - let new_traceback = - PyTraceback::new(next, frame.object.to_owned(), frame.lasti(), loc.row); - vm_trace!("Adding to traceback: {:?} {:?}", new_traceback, loc.row); + let new_traceback = PyTraceback::new( + next, + frame.object.to_owned(), + frame.lasti(), + loc.line, + ); + vm_trace!("Adding to traceback: {:?} {:?}", new_traceback, loc.line); exception.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); vm.contextualize_exception(&exception); diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 1533f903c5c..ab72c7170c0 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -21,6 +21,7 @@ use crate::{ }; use node::Node; use ruff_python_ast as ruff; +use ruff_source_file::PositionEncoding; use ruff_text_size::{Ranged, TextRange, TextSize}; use rustpython_compiler_core::{ LineIndex, OneIndexed, SourceFile, SourceFileBuilder, SourceLocation, @@ -92,8 +93,8 @@ pub struct PySourceLocation { impl PySourceLocation { const fn to_source_location(&self) -> SourceLocation { SourceLocation { - row: self.row.get_one_indexed(), - column: self.column.get_one_indexed(), + line: self.row.get_one_indexed(), + character_offset: self.column.get_one_indexed(), } } } @@ -194,14 +195,14 @@ fn source_range_to_text_range(source_file: &SourceFile, location: PySourceRange) } let start = index.offset( - location.start.row.get_one_indexed(), - location.start.column.get_one_indexed(), + location.start.to_source_location(), source, + PositionEncoding::Utf8, ); let end = index.offset( - location.end.row.get_one_indexed(), - location.end.column.get_one_indexed(), + location.end.to_source_location(), source, + PositionEncoding::Utf8, ); TextRange::new(start, end) @@ -273,9 +274,11 @@ pub(crate) fn compile( let ast: Mod = Node::ast_from_object(vm, &source_file, object)?; let ast = match ast { Mod::Module(m) => ruff::Mod::Module(m), - Mod::Interactive(ModInteractive { range, body }) => { - ruff::Mod::Module(ruff::ModModule { range, body }) - } + Mod::Interactive(ModInteractive { range, body }) => ruff::Mod::Module(ruff::ModModule { + range, + body, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + }), Mod::Expression(e) => ruff::Mod::Expression(e), Mod::FunctionType(_) => todo!(), }; diff --git a/vm/src/stdlib/ast/argument.rs b/vm/src/stdlib/ast/argument.rs index a1207f2c054..4568208251c 100644 --- a/vm/src/stdlib/ast/argument.rs +++ b/vm/src/stdlib/ast/argument.rs @@ -60,6 +60,7 @@ pub(super) fn merge_function_call_arguments( range, args: pos_args.args, keywords: key_args.keywords, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, } } @@ -70,6 +71,7 @@ pub(super) fn split_function_call_arguments( range: _, args, keywords, + node_index: _, } = args; let positional_arguments_range = args @@ -108,6 +110,7 @@ pub(super) fn split_class_def_args( range: _, args, keywords, + node_index: _, } = args; let positional_arguments_range = args @@ -158,5 +161,6 @@ pub(super) fn merge_class_def_args( range: Default::default(), // TODO args, keywords, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, })) } diff --git a/vm/src/stdlib/ast/constant.rs b/vm/src/stdlib/ast/constant.rs index b6c3e9596ba..cae2416607a 100644 --- a/vm/src/stdlib/ast/constant.rs +++ b/vm/src/stdlib/ast/constant.rs @@ -247,10 +247,15 @@ impl Node for ConstantLiteral { fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { let Constant { value, range } = value; match value { - ConstantLiteral::None => ruff::Expr::NoneLiteral(ruff::ExprNoneLiteral { range }), - ConstantLiteral::Bool(value) => { - ruff::Expr::BooleanLiteral(ruff::ExprBooleanLiteral { range, value }) - } + ConstantLiteral::None => ruff::Expr::NoneLiteral(ruff::ExprNoneLiteral { + range, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + }), + ConstantLiteral::Bool(value) => ruff::Expr::BooleanLiteral(ruff::ExprBooleanLiteral { + range, + value, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + }), ConstantLiteral::Str { value, prefix } => { ruff::Expr::StringLiteral(ruff::ExprStringLiteral { range, @@ -258,7 +263,9 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { range, value, flags: ruff::StringLiteralFlags::empty().with_prefix(prefix), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } ConstantLiteral::Bytes(value) => { @@ -268,12 +275,15 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { range, value, flags: ruff::BytesLiteralFlags::empty(), // TODO + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } ConstantLiteral::Int(value) => ruff::Expr::NumberLiteral(ruff::ExprNumberLiteral { range, value: ruff::Number::Int(value), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }), ConstantLiteral::Tuple(value) => ruff::Expr::Tuple(ruff::ExprTuple { range, @@ -289,6 +299,7 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { ctx: ruff::ExprContext::Load, // TODO: Does this matter? parenthesized: true, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }), ConstantLiteral::FrozenSet(value) => ruff::Expr::Call(ruff::ExprCall { range, @@ -297,6 +308,7 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { range: TextRange::default(), id: ruff::name::Name::new_static("frozenset"), ctx: ruff::ExprContext::Load, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, })), arguments: ruff::Arguments { range, @@ -310,21 +322,26 @@ fn constant_to_ruff_expr(value: Constant) -> ruff::Expr { }) .collect(), keywords: Box::default(), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }), ConstantLiteral::Float(value) => ruff::Expr::NumberLiteral(ruff::ExprNumberLiteral { range, value: ruff::Number::Float(value), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }), ConstantLiteral::Complex { real, imag } => { ruff::Expr::NumberLiteral(ruff::ExprNumberLiteral { range, value: ruff::Number::Complex { real, imag }, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } - ConstantLiteral::Ellipsis => { - ruff::Expr::EllipsisLiteral(ruff::ExprEllipsisLiteral { range }) - } + ConstantLiteral::Ellipsis => ruff::Expr::EllipsisLiteral(ruff::ExprEllipsisLiteral { + range, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + }), } } @@ -333,7 +350,11 @@ pub(super) fn number_literal_to_object( source_file: &SourceFile, constant: ruff::ExprNumberLiteral, ) -> PyObjectRef { - let ruff::ExprNumberLiteral { range, value } = constant; + let ruff::ExprNumberLiteral { + range, + value, + node_index: _, + } = constant; let c = match value { ruff::Number::Int(n) => Constant::new_int(n, range), ruff::Number::Float(n) => Constant::new_float(n, range), @@ -347,7 +368,11 @@ pub(super) fn string_literal_to_object( source_file: &SourceFile, constant: ruff::ExprStringLiteral, ) -> PyObjectRef { - let ruff::ExprStringLiteral { range, value } = constant; + let ruff::ExprStringLiteral { + range, + value, + node_index: _, + } = constant; let prefix = value .iter() .next() @@ -361,7 +386,11 @@ pub(super) fn bytes_literal_to_object( source_file: &SourceFile, constant: ruff::ExprBytesLiteral, ) -> PyObjectRef { - let ruff::ExprBytesLiteral { range, value } = constant; + let ruff::ExprBytesLiteral { + range, + value, + node_index: _, + } = constant; let bytes = value.as_slice().iter().flat_map(|b| b.value.iter()); let c = Constant::new_bytes(bytes.copied().collect(), range); c.ast_to_object(vm, source_file) @@ -372,7 +401,11 @@ pub(super) fn boolean_literal_to_object( source_file: &SourceFile, constant: ruff::ExprBooleanLiteral, ) -> PyObjectRef { - let ruff::ExprBooleanLiteral { range, value } = constant; + let ruff::ExprBooleanLiteral { + range, + value, + node_index: _, + } = constant; let c = Constant::new_bool(value, range); c.ast_to_object(vm, source_file) } @@ -382,7 +415,10 @@ pub(super) fn none_literal_to_object( source_file: &SourceFile, constant: ruff::ExprNoneLiteral, ) -> PyObjectRef { - let ruff::ExprNoneLiteral { range } = constant; + let ruff::ExprNoneLiteral { + range, + node_index: _, + } = constant; let c = Constant::new_none(range); c.ast_to_object(vm, source_file) } @@ -392,7 +428,10 @@ pub(super) fn ellipsis_literal_to_object( source_file: &SourceFile, constant: ruff::ExprEllipsisLiteral, ) -> PyObjectRef { - let ruff::ExprEllipsisLiteral { range } = constant; + let ruff::ExprEllipsisLiteral { + range, + node_index: _, + } = constant; let c = Constant::new_ellipsis(range); c.ast_to_object(vm, source_file) } diff --git a/vm/src/stdlib/ast/elif_else_clause.rs b/vm/src/stdlib/ast/elif_else_clause.rs index 2b427b1ec13..fe976458c1f 100644 --- a/vm/src/stdlib/ast/elif_else_clause.rs +++ b/vm/src/stdlib/ast/elif_else_clause.rs @@ -7,7 +7,12 @@ pub(super) fn ast_to_object( vm: &VirtualMachine, source_file: &SourceFile, ) -> PyObjectRef { - let ruff::ElifElseClause { range, test, body } = clause; + let ruff::ElifElseClause { + range, + test, + body, + node_index: _, + } = clause; let Some(test) = test else { assert!(rest.len() == 0); return body.ast_to_object(vm, source_file); @@ -59,6 +64,7 @@ pub(super) fn ast_from_object( test, body, mut elif_else_clauses, + node_index: _, })) = orelse.into_iter().next() else { unreachable!() @@ -69,6 +75,7 @@ pub(super) fn ast_from_object( range, test: Some(*test), body, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }, ); elif_else_clauses @@ -77,6 +84,7 @@ pub(super) fn ast_from_object( range, test: None, body: orelse, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }] }; @@ -85,5 +93,6 @@ pub(super) fn ast_from_object( body, elif_else_clauses, range, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } diff --git a/vm/src/stdlib/ast/exception.rs b/vm/src/stdlib/ast/exception.rs index df6b391aa1d..f3f1da9e040 100644 --- a/vm/src/stdlib/ast/exception.rs +++ b/vm/src/stdlib/ast/exception.rs @@ -39,6 +39,7 @@ impl Node for ruff::ExceptHandlerExceptHandler { name, body, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type( @@ -75,6 +76,7 @@ impl Node for ruff::ExceptHandlerExceptHandler { get_node_field(_vm, &_object, "body", "ExceptHandler")?, )?, range: range_from_object(_vm, source_file, _object, "ExceptHandler")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } diff --git a/vm/src/stdlib/ast/expression.rs b/vm/src/stdlib/ast/expression.rs index 31b65bd13a9..1ed848ddf54 100644 --- a/vm/src/stdlib/ast/expression.rs +++ b/vm/src/stdlib/ast/expression.rs @@ -45,6 +45,7 @@ impl Node for ruff::Expr { constant::ellipsis_literal_to_object(vm, source_file, cons) } Self::Named(cons) => cons.ast_to_object(vm, source_file), + Self::TString(_) => todo!(), Self::IpyEscapeCommand(_) => { unimplemented!("IPython escape command is not allowed in Python AST") } @@ -145,7 +146,12 @@ impl Node for ruff::Expr { // constructor impl Node for ruff::ExprBoolOp { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { op, values, range } = self; + let Self { + op, + values, + range, + node_index: _, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprBoolOp::static_type().to_owned()) .unwrap(); @@ -175,6 +181,7 @@ impl Node for ruff::ExprBoolOp { get_node_field(vm, &object, "values", "BoolOp")?, )?, range: range_from_object(vm, source_file, object, "BoolOp")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -186,6 +193,7 @@ impl Node for ruff::ExprNamed { target, value, range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprNamedExpr::static_type().to_owned()) @@ -216,6 +224,7 @@ impl Node for ruff::ExprNamed { get_node_field(vm, &object, "value", "NamedExpr")?, )?, range: range_from_object(vm, source_file, object, "NamedExpr")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -228,6 +237,7 @@ impl Node for ruff::ExprBinOp { op, right, range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprBinOp::static_type().to_owned()) @@ -265,6 +275,7 @@ impl Node for ruff::ExprBinOp { get_node_field(vm, &object, "right", "BinOp")?, )?, range: range_from_object(vm, source_file, object, "BinOp")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -272,7 +283,12 @@ impl Node for ruff::ExprBinOp { // constructor impl Node for ruff::ExprUnaryOp { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { op, operand, range } = self; + let Self { + op, + operand, + range, + node_index: _, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprUnaryOp::static_type().to_owned()) .unwrap(); @@ -301,6 +317,7 @@ impl Node for ruff::ExprUnaryOp { get_node_field(vm, &object, "operand", "UnaryOp")?, )?, range: range_from_object(vm, source_file, object, "UnaryOp")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -312,6 +329,7 @@ impl Node for ruff::ExprLambda { parameters, body, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprLambda::static_type().to_owned()) @@ -342,6 +360,7 @@ impl Node for ruff::ExprLambda { get_node_field(vm, &object, "body", "Lambda")?, )?, range: range_from_object(vm, source_file, object, "Lambda")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -354,6 +373,7 @@ impl Node for ruff::ExprIf { body, orelse, range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprIfExp::static_type().to_owned()) @@ -391,6 +411,7 @@ impl Node for ruff::ExprIf { get_node_field(vm, &object, "orelse", "IfExp")?, )?, range: range_from_object(vm, source_file, object, "IfExp")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -398,7 +419,11 @@ impl Node for ruff::ExprIf { // constructor impl Node for ruff::ExprDict { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { items, range } = self; + let Self { + items, + range, + node_index: _, + } = self; let (keys, values) = items .into_iter() @@ -442,6 +467,7 @@ impl Node for ruff::ExprDict { Ok(Self { items, range: range_from_object(vm, source_file, object, "Dict")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -449,7 +475,11 @@ impl Node for ruff::ExprDict { // constructor impl Node for ruff::ExprSet { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { elts, range } = self; + let Self { + elts, + range, + node_index: _, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprSet::static_type().to_owned()) .unwrap(); @@ -471,6 +501,7 @@ impl Node for ruff::ExprSet { get_node_field(vm, &object, "elts", "Set")?, )?, range: range_from_object(vm, source_file, object, "Set")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -482,6 +513,7 @@ impl Node for ruff::ExprListComp { elt, generators, range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprListComp::static_type().to_owned()) @@ -512,6 +544,7 @@ impl Node for ruff::ExprListComp { get_node_field(vm, &object, "generators", "ListComp")?, )?, range: range_from_object(vm, source_file, object, "ListComp")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -523,6 +556,7 @@ impl Node for ruff::ExprSetComp { elt, generators, range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprSetComp::static_type().to_owned()) @@ -553,6 +587,7 @@ impl Node for ruff::ExprSetComp { get_node_field(vm, &object, "generators", "SetComp")?, )?, range: range_from_object(vm, source_file, object, "SetComp")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -565,6 +600,7 @@ impl Node for ruff::ExprDictComp { value, generators, range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprDictComp::static_type().to_owned()) @@ -602,6 +638,7 @@ impl Node for ruff::ExprDictComp { get_node_field(vm, &object, "generators", "DictComp")?, )?, range: range_from_object(vm, source_file, object, "DictComp")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -614,6 +651,7 @@ impl Node for ruff::ExprGenerator { generators, range, parenthesized: _, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprGeneratorExp::static_type().to_owned()) @@ -646,6 +684,7 @@ impl Node for ruff::ExprGenerator { range: range_from_object(vm, source_file, object, "GeneratorExp")?, // TODO: Is this correct? parenthesized: true, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -653,7 +692,11 @@ impl Node for ruff::ExprGenerator { // constructor impl Node for ruff::ExprAwait { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { value, range } = self; + let Self { + value, + range, + node_index: _, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprAwait::static_type().to_owned()) .unwrap(); @@ -675,6 +718,7 @@ impl Node for ruff::ExprAwait { get_node_field(vm, &object, "value", "Await")?, )?, range: range_from_object(vm, source_file, object, "Await")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -682,7 +726,11 @@ impl Node for ruff::ExprAwait { // constructor impl Node for ruff::ExprYield { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { value, range } = self; + let Self { + value, + range, + node_index: _, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprYield::static_type().to_owned()) .unwrap(); @@ -703,6 +751,7 @@ impl Node for ruff::ExprYield { .map(|obj| Node::ast_from_object(vm, source_file, obj)) .transpose()?, range: range_from_object(vm, source_file, object, "Yield")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -710,7 +759,11 @@ impl Node for ruff::ExprYield { // constructor impl Node for ruff::ExprYieldFrom { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { value, range } = self; + let Self { + value, + range, + node_index: _, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprYieldFrom::static_type().to_owned()) .unwrap(); @@ -733,6 +786,7 @@ impl Node for ruff::ExprYieldFrom { get_node_field(vm, &object, "value", "YieldFrom")?, )?, range: range_from_object(vm, source_file, object, "YieldFrom")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -745,6 +799,7 @@ impl Node for ruff::ExprCompare { ops, comparators, range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprCompare::static_type().to_owned()) @@ -792,6 +847,7 @@ impl Node for ruff::ExprCompare { comparators.0 }, range: range_from_object(vm, source_file, object, "Compare")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -803,6 +859,7 @@ impl Node for ruff::ExprCall { func, arguments, range, + node_index: _, } = self; let (positional_arguments, keyword_arguments) = split_function_call_arguments(arguments); let node = NodeAst @@ -851,6 +908,7 @@ impl Node for ruff::ExprCall { )?, ), range: range_from_object(vm, source_file, object, "Call")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -863,6 +921,7 @@ impl Node for ruff::ExprAttribute { attr, ctx, range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprAttribute::static_type().to_owned()) @@ -900,6 +959,7 @@ impl Node for ruff::ExprAttribute { get_node_field(vm, &object, "ctx", "Attribute")?, )?, range: range_from_object(vm, source_file, object, "Attribute")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -912,6 +972,7 @@ impl Node for ruff::ExprSubscript { slice, ctx, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprSubscript::static_type().to_owned()) @@ -948,6 +1009,7 @@ impl Node for ruff::ExprSubscript { get_node_field(vm, &object, "ctx", "Subscript")?, )?, range: range_from_object(vm, source_file, object, "Subscript")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -955,7 +1017,12 @@ impl Node for ruff::ExprSubscript { // constructor impl Node for ruff::ExprStarred { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { value, ctx, range } = self; + let Self { + value, + ctx, + range, + node_index: _, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprStarred::static_type().to_owned()) .unwrap(); @@ -984,6 +1051,7 @@ impl Node for ruff::ExprStarred { get_node_field(vm, &object, "ctx", "Starred")?, )?, range: range_from_object(vm, source_file, object, "Starred")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -991,7 +1059,12 @@ impl Node for ruff::ExprStarred { // constructor impl Node for ruff::ExprName { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { id, ctx, range } = self; + let Self { + id, + ctx, + range, + node_index: _, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprName::static_type().to_owned()) .unwrap(); @@ -1016,6 +1089,7 @@ impl Node for ruff::ExprName { get_node_field(vm, &object, "ctx", "Name")?, )?, range: range_from_object(vm, source_file, object, "Name")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -1023,7 +1097,12 @@ impl Node for ruff::ExprName { // constructor impl Node for ruff::ExprList { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { elts, ctx, range } = self; + let Self { + elts, + ctx, + range, + node_index: _, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprList::static_type().to_owned()) .unwrap(); @@ -1053,6 +1132,7 @@ impl Node for ruff::ExprList { get_node_field(vm, &object, "ctx", "List")?, )?, range: range_from_object(vm, source_file, object, "List")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -1065,6 +1145,7 @@ impl Node for ruff::ExprTuple { ctx, range: _range, parenthesized: _, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprTuple::static_type().to_owned()) @@ -1096,6 +1177,7 @@ impl Node for ruff::ExprTuple { )?, range: range_from_object(vm, source_file, object, "Tuple")?, parenthesized: true, // TODO: is this correct? + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -1108,6 +1190,7 @@ impl Node for ruff::ExprSlice { upper, step, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeExprSlice::static_type().to_owned()) @@ -1139,6 +1222,7 @@ impl Node for ruff::ExprSlice { .map(|obj| Node::ast_from_object(vm, source_file, obj)) .transpose()?, range: range_from_object(vm, source_file, object, "Slice")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -1190,6 +1274,7 @@ impl Node for ruff::Comprehension { ifs, is_async, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeComprehension::static_type().to_owned()) @@ -1233,6 +1318,7 @@ impl Node for ruff::Comprehension { get_node_field(vm, &object, "is_async", "comprehension")?, )?, range: Default::default(), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } diff --git a/vm/src/stdlib/ast/module.rs b/vm/src/stdlib/ast/module.rs index 33ee4c567b1..a7bc5b16031 100644 --- a/vm/src/stdlib/ast/module.rs +++ b/vm/src/stdlib/ast/module.rs @@ -69,6 +69,7 @@ impl Node for ruff::ModModule { body, // type_ignores, range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeModModule::static_type().to_owned()) @@ -105,6 +106,7 @@ impl Node for ruff::ModModule { // get_node_field(_vm, &_object, "type_ignores", "Module")?, // )?, range: Default::default(), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -147,7 +149,11 @@ impl Node for ModInteractive { // constructor impl Node for ruff::ModExpression { fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { body, range } = self; + let Self { + body, + range, + node_index: _, + } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeModExpression::static_type().to_owned()) .unwrap(); @@ -170,6 +176,7 @@ impl Node for ruff::ModExpression { get_node_field(vm, &object, "body", "Expression")?, )?, range: Default::default(), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } diff --git a/vm/src/stdlib/ast/other.rs b/vm/src/stdlib/ast/other.rs index eddbef97482..257793d0b68 100644 --- a/vm/src/stdlib/ast/other.rs +++ b/vm/src/stdlib/ast/other.rs @@ -55,7 +55,11 @@ impl Node for ruff::Decorator { ) -> PyResult { let expression = ruff::Expr::ast_from_object(vm, source_file, object)?; let range = expression.range(); - Ok(Self { expression, range }) + Ok(Self { + expression, + range, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + }) } } @@ -66,6 +70,7 @@ impl Node for ruff::Alias { name, asname, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeAlias::static_type().to_owned()) @@ -94,6 +99,7 @@ impl Node for ruff::Alias { .map(|obj| Node::ast_from_object(vm, source_file, obj)) .transpose()?, range: range_from_object(vm, source_file, object, "alias")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -105,6 +111,7 @@ impl Node for ruff::WithItem { context_expr, optional_vars, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeWithItem::static_type().to_owned()) @@ -140,6 +147,7 @@ impl Node for ruff::WithItem { .map(|obj| Node::ast_from_object(vm, source_file, obj)) .transpose()?, range: Default::default(), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } diff --git a/vm/src/stdlib/ast/parameter.rs b/vm/src/stdlib/ast/parameter.rs index 347b3b4ec0c..8e5bdc9a943 100644 --- a/vm/src/stdlib/ast/parameter.rs +++ b/vm/src/stdlib/ast/parameter.rs @@ -11,6 +11,7 @@ impl Node for ruff::Parameters { kwonlyargs, kwarg, range, + node_index: _, } = self; let (posonlyargs, args, defaults) = extract_positional_parameter_defaults(posonlyargs, args); @@ -90,6 +91,7 @@ impl Node for ruff::Parameters { .map(|obj| Node::ast_from_object(vm, source_file, obj)) .transpose()?, range: Default::default(), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } @@ -106,6 +108,7 @@ impl Node for ruff::Parameter { annotation, // type_comment, range, + node_index: _, } = self; // ruff covers the ** in range but python expects it to start at the ident @@ -147,6 +150,7 @@ impl Node for ruff::Parameter { // .map(|obj| Node::ast_from_object(_vm, obj)) // .transpose()?, range: range_from_object(_vm, source_file, _object, "arg")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -158,6 +162,7 @@ impl Node for ruff::Keyword { arg, value, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeKeyword::static_type().to_owned()) @@ -170,6 +175,7 @@ impl Node for ruff::Keyword { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -185,6 +191,7 @@ impl Node for ruff::Keyword { get_node_field(_vm, &_object, "value", "keyword")?, )?, range: range_from_object(_vm, source_file, _object, "keyword")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -331,6 +338,7 @@ fn merge_positional_parameter_defaults( range: Default::default(), parameter, default: None, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) .collect(); let mut args: Vec<_> = as IntoIterator>::into_iter(args) @@ -338,6 +346,7 @@ fn merge_positional_parameter_defaults( range: Default::default(), parameter, default: None, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) .collect(); @@ -400,6 +409,7 @@ fn merge_keyword_parameter_defaults( parameter, default, range: Default::default(), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) .collect() } diff --git a/vm/src/stdlib/ast/pattern.rs b/vm/src/stdlib/ast/pattern.rs index 0c484d6787a..ed452ac0cf5 100644 --- a/vm/src/stdlib/ast/pattern.rs +++ b/vm/src/stdlib/ast/pattern.rs @@ -9,6 +9,7 @@ impl Node for ruff::MatchCase { guard, body, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeMatchCase::static_type().to_owned()) @@ -43,6 +44,7 @@ impl Node for ruff::MatchCase { get_node_field(_vm, &_object, "body", "match_case")?, )?, range: Default::default(), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -129,6 +131,7 @@ impl Node for ruff::PatternMatchValue { let Self { value, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodePatternMatchValue::static_type().to_owned()) @@ -152,6 +155,7 @@ impl Node for ruff::PatternMatchValue { get_node_field(_vm, &_object, "value", "MatchValue")?, )?, range: range_from_object(_vm, source_file, _object, "MatchValue")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -162,6 +166,7 @@ impl Node for ruff::PatternMatchSingleton { let Self { value, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type( @@ -188,6 +193,7 @@ impl Node for ruff::PatternMatchSingleton { get_node_field(_vm, &_object, "value", "MatchSingleton")?, )?, range: range_from_object(_vm, source_file, _object, "MatchSingleton")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -212,6 +218,7 @@ impl Node for ruff::PatternMatchSequence { let Self { patterns, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type( @@ -238,6 +245,7 @@ impl Node for ruff::PatternMatchSequence { get_node_field(_vm, &_object, "patterns", "MatchSequence")?, )?, range: range_from_object(_vm, source_file, _object, "MatchSequence")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -250,6 +258,7 @@ impl Node for ruff::PatternMatchMapping { patterns, rest, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type( @@ -288,6 +297,7 @@ impl Node for ruff::PatternMatchMapping { .map(|obj| Node::ast_from_object(_vm, source_file, obj)) .transpose()?, range: range_from_object(_vm, source_file, _object, "MatchMapping")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -299,6 +309,7 @@ impl Node for ruff::PatternMatchClass { cls, arguments, range: _range, + node_index: _, } = self; let (patterns, kwd_attrs, kwd_patterns) = split_pattern_match_class(arguments); let node = NodeAst @@ -354,7 +365,9 @@ impl Node for ruff::PatternMatchClass { range: Default::default(), patterns, keywords, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -412,12 +425,14 @@ impl Node for PatternMatchClassKeywordPatterns { todo!() } } + // constructor impl Node for ruff::PatternMatchStar { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { name, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodePatternMatchStar::static_type().to_owned()) @@ -439,6 +454,7 @@ impl Node for ruff::PatternMatchStar { .map(|obj| Node::ast_from_object(_vm, source_file, obj)) .transpose()?, range: range_from_object(_vm, source_file, _object, "MatchStar")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -450,6 +466,7 @@ impl Node for ruff::PatternMatchAs { pattern, name, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodePatternMatchAs::static_type().to_owned()) @@ -476,6 +493,7 @@ impl Node for ruff::PatternMatchAs { .map(|obj| Node::ast_from_object(_vm, source_file, obj)) .transpose()?, range: range_from_object(_vm, source_file, _object, "MatchAs")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -486,6 +504,7 @@ impl Node for ruff::PatternMatchOr { let Self { patterns, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodePatternMatchOr::static_type().to_owned()) @@ -496,6 +515,7 @@ impl Node for ruff::PatternMatchOr { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -508,6 +528,7 @@ impl Node for ruff::PatternMatchOr { get_node_field(_vm, &_object, "patterns", "MatchOr")?, )?, range: range_from_object(_vm, source_file, _object, "MatchOr")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } diff --git a/vm/src/stdlib/ast/statement.rs b/vm/src/stdlib/ast/statement.rs index fab7ea76961..43a5ad2a07b 100644 --- a/vm/src/stdlib/ast/statement.rs +++ b/vm/src/stdlib/ast/statement.rs @@ -181,6 +181,7 @@ impl Node for ruff::StmtFunctionDef { type_params, is_async, range: _range, + node_index: _, } = self; let cls = if !is_async { @@ -217,6 +218,7 @@ impl Node for ruff::StmtFunctionDef { node_add_location(&dict, _range, vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -259,6 +261,7 @@ impl Node for ruff::StmtFunctionDef { )?, range: range_from_object(_vm, source_file, _object, "FunctionDef")?, is_async, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -273,6 +276,7 @@ impl Node for ruff::StmtClassDef { decorator_list, type_params, range: _range, + node_index: _, } = self; let (bases, keywords) = split_class_def_args(arguments); let node = NodeAst @@ -302,6 +306,7 @@ impl Node for ruff::StmtClassDef { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -340,15 +345,18 @@ impl Node for ruff::StmtClassDef { get_node_field(_vm, &_object, "type_params", "ClassDef")?, )?, range: range_from_object(_vm, source_file, _object, "ClassDef")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } + // constructor impl Node for ruff::StmtReturn { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { value, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtReturn::static_type().to_owned()) @@ -359,6 +367,7 @@ impl Node for ruff::StmtReturn { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -369,15 +378,18 @@ impl Node for ruff::StmtReturn { .map(|obj| Node::ast_from_object(_vm, source_file, obj)) .transpose()?, range: range_from_object(_vm, source_file, _object, "Return")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } + // constructor impl Node for ruff::StmtDelete { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { let Self { targets, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtDelete::static_type().to_owned()) @@ -388,6 +400,7 @@ impl Node for ruff::StmtDelete { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -400,6 +413,7 @@ impl Node for ruff::StmtDelete { get_node_field(_vm, &_object, "targets", "Delete")?, )?, range: range_from_object(_vm, source_file, _object, "Delete")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -412,6 +426,7 @@ impl Node for ruff::StmtAssign { value, // type_comment, range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeStmtAssign::static_type().to_owned()) @@ -426,6 +441,7 @@ impl Node for ruff::StmtAssign { node_add_location(&dict, range, vm, source_file); node.into() } + fn ast_from_object( vm: &VirtualMachine, source_file: &SourceFile, @@ -446,6 +462,7 @@ impl Node for ruff::StmtAssign { // .map(|obj| Node::ast_from_object(_vm, obj)) // .transpose()?, range: range_from_object(vm, source_file, object, "Assign")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -458,6 +475,7 @@ impl Node for ruff::StmtTypeAlias { type_params, value, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtTypeAlias::static_type().to_owned()) @@ -499,6 +517,7 @@ impl Node for ruff::StmtTypeAlias { get_node_field(_vm, &_object, "value", "TypeAlias")?, )?, range: range_from_object(_vm, source_file, _object, "TypeAlias")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -511,6 +530,7 @@ impl Node for ruff::StmtAugAssign { op, value, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtAugAssign::static_type().to_owned()) @@ -525,6 +545,7 @@ impl Node for ruff::StmtAugAssign { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -547,6 +568,7 @@ impl Node for ruff::StmtAugAssign { get_node_field(_vm, &_object, "value", "AugAssign")?, )?, range: range_from_object(_vm, source_file, _object, "AugAssign")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -560,6 +582,7 @@ impl Node for ruff::StmtAnnAssign { value, simple, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtAnnAssign::static_type().to_owned()) @@ -580,6 +603,7 @@ impl Node for ruff::StmtAnnAssign { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -605,6 +629,7 @@ impl Node for ruff::StmtAnnAssign { get_node_field(_vm, &_object, "simple", "AnnAssign")?, )?, range: range_from_object(_vm, source_file, _object, "AnnAssign")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -620,6 +645,7 @@ impl Node for ruff::StmtFor { orelse, // type_comment, range: _range, + node_index: _, } = self; let cls = if !is_async { @@ -681,6 +707,7 @@ impl Node for ruff::StmtFor { // .transpose()?, range: range_from_object(_vm, source_file, _object, "For")?, is_async, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -693,6 +720,7 @@ impl Node for ruff::StmtWhile { body, orelse, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtWhile::static_type().to_owned()) @@ -730,9 +758,11 @@ impl Node for ruff::StmtWhile { get_node_field(_vm, &_object, "orelse", "While")?, )?, range: range_from_object(_vm, source_file, _object, "While")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } + // constructor impl Node for ruff::StmtIf { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { @@ -741,18 +771,21 @@ impl Node for ruff::StmtIf { body, range, elif_else_clauses, + node_index: _, } = self; elif_else_clause::ast_to_object( ruff::ElifElseClause { range, test: Some(*test), body, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }, elif_else_clauses.into_iter(), _vm, source_file, ) } + fn ast_from_object( vm: &VirtualMachine, source_file: &SourceFile, @@ -761,6 +794,7 @@ impl Node for ruff::StmtIf { elif_else_clause::ast_from_object(vm, source_file, object) } } + // constructor impl Node for ruff::StmtWith { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { @@ -770,6 +804,7 @@ impl Node for ruff::StmtWith { body, // type_comment, range: _range, + node_index: _, } = self; let cls = if !is_async { @@ -789,6 +824,7 @@ impl Node for ruff::StmtWith { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -816,9 +852,11 @@ impl Node for ruff::StmtWith { // .transpose()?, range: range_from_object(_vm, source_file, _object, "With")?, is_async, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } + // constructor impl Node for ruff::StmtMatch { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { @@ -826,6 +864,7 @@ impl Node for ruff::StmtMatch { subject, cases, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtMatch::static_type().to_owned()) @@ -838,6 +877,7 @@ impl Node for ruff::StmtMatch { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -855,9 +895,11 @@ impl Node for ruff::StmtMatch { get_node_field(_vm, &_object, "cases", "Match")?, )?, range: range_from_object(_vm, source_file, _object, "Match")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } + // constructor impl Node for ruff::StmtRaise { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { @@ -865,6 +907,7 @@ impl Node for ruff::StmtRaise { exc, cause, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtRaise::static_type().to_owned()) @@ -877,6 +920,7 @@ impl Node for ruff::StmtRaise { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -890,9 +934,11 @@ impl Node for ruff::StmtRaise { .map(|obj| Node::ast_from_object(_vm, source_file, obj)) .transpose()?, range: range_from_object(_vm, source_file, _object, "Raise")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } + // constructor impl Node for ruff::StmtTry { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { @@ -903,6 +949,7 @@ impl Node for ruff::StmtTry { finalbody, range: _range, is_star, + node_index: _, } = self; // let cls = gen::NodeStmtTry::static_type().to_owned(); @@ -926,6 +973,7 @@ impl Node for ruff::StmtTry { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -962,6 +1010,7 @@ impl Node for ruff::StmtTry { )?, range: range_from_object(_vm, source_file, _object, "Try")?, is_star, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -972,6 +1021,7 @@ impl Node for ruff::StmtAssert { test, msg, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtAssert::static_type().to_owned()) @@ -984,6 +1034,7 @@ impl Node for ruff::StmtAssert { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -999,6 +1050,7 @@ impl Node for ruff::StmtAssert { .map(|obj| Node::ast_from_object(_vm, source_file, obj)) .transpose()?, range: range_from_object(_vm, source_file, _object, "Assert")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -1008,6 +1060,7 @@ impl Node for ruff::StmtImport { let Self { names, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtImport::static_type().to_owned()) @@ -1018,6 +1071,7 @@ impl Node for ruff::StmtImport { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -1030,6 +1084,7 @@ impl Node for ruff::StmtImport { get_node_field(_vm, &_object, "names", "Import")?, )?, range: range_from_object(_vm, source_file, _object, "Import")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -1041,6 +1096,7 @@ impl Node for ruff::StmtImportFrom { names, level, range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeStmtImportFrom::static_type().to_owned()) @@ -1055,6 +1111,7 @@ impl Node for ruff::StmtImportFrom { node_add_location(&dict, range, vm, source_file); node.into() } + fn ast_from_object( vm: &VirtualMachine, source_file: &SourceFile, @@ -1074,6 +1131,7 @@ impl Node for ruff::StmtImportFrom { .unwrap() .try_to_primitive::(vm)?, range: range_from_object(vm, source_file, _object, "ImportFrom")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -1083,6 +1141,7 @@ impl Node for ruff::StmtGlobal { let Self { names, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtGlobal::static_type().to_owned()) @@ -1093,6 +1152,7 @@ impl Node for ruff::StmtGlobal { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -1105,6 +1165,7 @@ impl Node for ruff::StmtGlobal { get_node_field(_vm, &_object, "names", "Global")?, )?, range: range_from_object(_vm, source_file, _object, "Global")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -1114,6 +1175,7 @@ impl Node for ruff::StmtNonlocal { let Self { names, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtNonlocal::static_type().to_owned()) @@ -1124,6 +1186,7 @@ impl Node for ruff::StmtNonlocal { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -1136,6 +1199,7 @@ impl Node for ruff::StmtNonlocal { get_node_field(_vm, &_object, "names", "Nonlocal")?, )?, range: range_from_object(_vm, source_file, _object, "Nonlocal")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -1145,6 +1209,7 @@ impl Node for ruff::StmtExpr { let Self { value, range: _range, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtExpr::static_type().to_owned()) @@ -1155,6 +1220,7 @@ impl Node for ruff::StmtExpr { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -1167,13 +1233,17 @@ impl Node for ruff::StmtExpr { get_node_field(_vm, &_object, "value", "Expr")?, )?, range: range_from_object(_vm, source_file, _object, "Expr")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } // constructor impl Node for ruff::StmtPass { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { range: _range } = self; + let Self { + range: _range, + node_index: _, + } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtPass::static_type().to_owned()) .unwrap(); @@ -1181,6 +1251,7 @@ impl Node for ruff::StmtPass { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -1188,13 +1259,17 @@ impl Node for ruff::StmtPass { ) -> PyResult { Ok(Self { range: range_from_object(_vm, source_file, _object, "Pass")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } // constructor impl Node for ruff::StmtBreak { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { range: _range } = self; + let Self { + range: _range, + node_index: _, + } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtBreak::static_type().to_owned()) .unwrap(); @@ -1210,6 +1285,7 @@ impl Node for ruff::StmtBreak { ) -> PyResult { Ok(Self { range: range_from_object(_vm, source_file, _object, "Break")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -1217,7 +1293,10 @@ impl Node for ruff::StmtBreak { // constructor impl Node for ruff::StmtContinue { fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { - let Self { range: _range } = self; + let Self { + range: _range, + node_index: _, + } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeStmtContinue::static_type().to_owned()) .unwrap(); @@ -1225,6 +1304,7 @@ impl Node for ruff::StmtContinue { node_add_location(&dict, _range, _vm, source_file); node.into() } + fn ast_from_object( _vm: &VirtualMachine, source_file: &SourceFile, @@ -1232,6 +1312,7 @@ impl Node for ruff::StmtContinue { ) -> PyResult { Ok(Self { range: range_from_object(_vm, source_file, _object, "Continue")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } diff --git a/vm/src/stdlib/ast/string.rs b/vm/src/stdlib/ast/string.rs index 5d8654270d4..3779f061a4d 100644 --- a/vm/src/stdlib/ast/string.rs +++ b/vm/src/stdlib/ast/string.rs @@ -8,6 +8,7 @@ fn ruff_fstring_value_into_iter( range: Default::default(), elements: Default::default(), flags: ruff::FStringFlags::empty(), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }); (0..fstring_value.as_slice().len()).map(move |i| { let fstring_value = &mut fstring_value; @@ -17,12 +18,14 @@ fn ruff_fstring_value_into_iter( } fn ruff_fstring_element_into_iter( - mut fstring_element: ruff::FStringElements, -) -> impl Iterator + 'static { - let default = ruff::FStringElement::Literal(ruff::FStringLiteralElement { - range: Default::default(), - value: Default::default(), - }); + mut fstring_element: ruff::InterpolatedStringElements, +) -> impl Iterator + 'static { + let default = + ruff::InterpolatedStringElement::Literal(ruff::InterpolatedStringLiteralElement { + range: Default::default(), + value: Default::default(), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + }); (0..fstring_element.into_iter().len()).map(move |i| { let fstring_element = &mut fstring_element; let tmp = fstring_element.into_iter().nth(i).unwrap(); @@ -36,6 +39,7 @@ fn fstring_part_to_joined_str_part(fstring_part: ruff::FStringPart) -> Vec { vec![JoinedStrPart::Constant(Constant::new_str( value, @@ -47,27 +51,33 @@ fn fstring_part_to_joined_str_part(fstring_part: ruff::FStringPart) -> Vec ruff_fstring_element_into_iter(elements) .map(ruff_fstring_element_to_joined_str_part) .collect(), } } -fn ruff_fstring_element_to_joined_str_part(element: ruff::FStringElement) -> JoinedStrPart { +fn ruff_fstring_element_to_joined_str_part( + element: ruff::InterpolatedStringElement, +) -> JoinedStrPart { match element { - ruff::FStringElement::Literal(ruff::FStringLiteralElement { range, value }) => { - JoinedStrPart::Constant(Constant::new_str( - value, - ruff::str_prefix::StringLiteralPrefix::Empty, - range, - )) - } - ruff::FStringElement::Expression(ruff::FStringExpressionElement { + ruff::InterpolatedStringElement::Literal(ruff::InterpolatedStringLiteralElement { + range, + value, + node_index: _, + }) => JoinedStrPart::Constant(Constant::new_str( + value, + ruff::str_prefix::StringLiteralPrefix::Empty, + range, + )), + ruff::InterpolatedStringElement::Interpolation(ruff::InterpolatedElement { range, expression, debug_text: _, // TODO: What is this? conversion, format_spec, + node_index: _, }) => JoinedStrPart::FormattedValue(FormattedValue { value: expression, conversion, @@ -78,12 +88,16 @@ fn ruff_fstring_element_to_joined_str_part(element: ruff::FStringElement) -> Joi } fn ruff_format_spec_to_joined_str( - format_spec: Option>, + format_spec: Option>, ) -> Option> { match format_spec { None => None, Some(format_spec) => { - let ruff::FStringFormatSpec { range, elements } = *format_spec; + let ruff::InterpolatedStringFormatSpec { + range, + elements, + node_index: _, + } = *format_spec; let values: Vec<_> = ruff_fstring_element_into_iter(elements) .map(ruff_fstring_element_to_joined_str_part) .collect(); @@ -93,37 +107,47 @@ fn ruff_format_spec_to_joined_str( } } -fn ruff_fstring_element_to_ruff_fstring_part(element: ruff::FStringElement) -> ruff::FStringPart { +fn ruff_fstring_element_to_ruff_fstring_part( + element: ruff::InterpolatedStringElement, +) -> ruff::FStringPart { match element { - ruff::FStringElement::Literal(value) => { - let ruff::FStringLiteralElement { range, value } = value; + ruff::InterpolatedStringElement::Literal(value) => { + let ruff::InterpolatedStringLiteralElement { + range, + value, + node_index: _, + } = value; ruff::FStringPart::Literal(ruff::StringLiteral { range, value, flags: ruff::StringLiteralFlags::empty(), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } - ruff::FStringElement::Expression(value) => { - let ruff::FStringExpressionElement { + ruff::InterpolatedStringElement::Interpolation(value) => { + let ruff::InterpolatedElement { range, expression, debug_text, conversion, format_spec, + node_index: _, } = value; ruff::FStringPart::FString(ruff::FString { range, - elements: vec![ruff::FStringElement::Expression( - ruff::FStringExpressionElement { + elements: vec![ruff::InterpolatedStringElement::Interpolation( + ruff::InterpolatedElement { range, expression, debug_text, conversion, format_spec, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }, )] .into(), flags: ruff::FStringFlags::empty(), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -131,7 +155,7 @@ fn ruff_fstring_element_to_ruff_fstring_part(element: ruff::FStringElement) -> r fn joined_str_to_ruff_format_spec( joined_str: Option>, -) -> Option> { +) -> Option> { match joined_str { None => None, Some(joined_str) => { @@ -139,9 +163,10 @@ fn joined_str_to_ruff_format_spec( let elements: Vec<_> = Box::into_iter(values) .map(joined_str_part_to_ruff_fstring_element) .collect(); - let format_spec = ruff::FStringFormatSpec { + let format_spec = ruff::InterpolatedStringFormatSpec { range, elements: elements.into(), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }; Some(Box::new(format_spec)) } @@ -165,6 +190,7 @@ impl JoinedStr { range, elements: vec![].into(), flags: ruff::FStringFlags::empty(), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }), 1 => ruff::FStringValue::single( Box::<[_]>::into_iter(values) @@ -173,6 +199,7 @@ impl JoinedStr { range, elements: vec![element].into(), flags: ruff::FStringFlags::empty(), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) .next() .expect("FString has exactly one part"), @@ -184,28 +211,31 @@ impl JoinedStr { .collect(), ), }, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } -fn joined_str_part_to_ruff_fstring_element(part: JoinedStrPart) -> ruff::FStringElement { +fn joined_str_part_to_ruff_fstring_element(part: JoinedStrPart) -> ruff::InterpolatedStringElement { match part { JoinedStrPart::FormattedValue(value) => { - ruff::FStringElement::Expression(ruff::FStringExpressionElement { + ruff::InterpolatedStringElement::Interpolation(ruff::InterpolatedElement { range: value.range, expression: value.value.clone(), debug_text: None, // TODO: What is this? conversion: value.conversion, format_spec: joined_str_to_ruff_format_spec(value.format_spec), + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } JoinedStrPart::Constant(value) => { - ruff::FStringElement::Literal(ruff::FStringLiteralElement { + ruff::InterpolatedStringElement::Literal(ruff::InterpolatedStringLiteralElement { range: value.range, value: match value.value { ConstantLiteral::Str { value, .. } => value, _ => todo!(), }, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -344,7 +374,11 @@ pub(super) fn fstring_to_object( source_file: &SourceFile, expression: ruff::ExprFString, ) -> PyObjectRef { - let ruff::ExprFString { range, value } = expression; + let ruff::ExprFString { + range, + value, + node_index: _, + } = expression; let values: Vec<_> = ruff_fstring_value_into_iter(value) .flat_map(fstring_part_to_joined_str_part) .collect(); diff --git a/vm/src/stdlib/ast/type_parameters.rs b/vm/src/stdlib/ast/type_parameters.rs index 505cd04d284..faf84823b6d 100644 --- a/vm/src/stdlib/ast/type_parameters.rs +++ b/vm/src/stdlib/ast/type_parameters.rs @@ -15,7 +15,11 @@ impl Node for ruff::TypeParams { let range = Option::zip(type_params.first(), type_params.last()) .map(|(first, last)| first.range().cover(last.range())) .unwrap_or_default(); - Ok(Self { type_params, range }) + Ok(Self { + type_params, + range, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, + }) } fn is_none(&self) -> bool { @@ -74,6 +78,7 @@ impl Node for ruff::TypeParamTypeVar { bound, range: _range, default: _, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeTypeParamTypeVar::static_type().to_owned()) @@ -107,6 +112,7 @@ impl Node for ruff::TypeParamTypeVar { get_node_field(_vm, &_object, "default_value", "TypeVar")?, )?, range: range_from_object(_vm, source_file, _object, "TypeVar")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -118,6 +124,7 @@ impl Node for ruff::TypeParamParamSpec { name, range: _range, default, + node_index: _, } = self; let node = NodeAst .into_ref_with_type(_vm, pyast::NodeTypeParamParamSpec::static_type().to_owned()) @@ -152,6 +159,7 @@ impl Node for ruff::TypeParamParamSpec { get_node_field(_vm, &_object, "default_value", "ParamSpec")?, )?, range: range_from_object(_vm, source_file, _object, "ParamSpec")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } @@ -163,6 +171,7 @@ impl Node for ruff::TypeParamTypeVarTuple { name, range: _range, default, + node_index: _, } = self; let node = NodeAst .into_ref_with_type( @@ -200,6 +209,7 @@ impl Node for ruff::TypeParamTypeVarTuple { get_node_field(_vm, &_object, "default_value", "TypeVarTuple")?, )?, range: range_from_object(_vm, source_file, _object, "TypeVarTuple")?, + node_index: ruff_python_ast::AtomicNodeIndex::NONE, }) } } diff --git a/vm/src/vm/vm_new.rs b/vm/src/vm/vm_new.rs index cf8d682ec24..d0b78cfe5bd 100644 --- a/vm/src/vm/vm_new.rs +++ b/vm/src/vm/vm_new.rs @@ -321,7 +321,7 @@ impl VirtualMachine { error: ruff_python_parser::ParseErrorType::Lexical( ruff_python_parser::LexicalErrorType::FStringError( - ruff_python_parser::FStringErrorType::UnterminatedTripleQuotedString, + ruff_python_parser::InterpolatedStringErrorType::UnterminatedTripleQuotedString, ), ), .. @@ -412,7 +412,7 @@ impl VirtualMachine { fn get_statement(source: &str, loc: Option) -> Option { let line = source .split('\n') - .nth(loc?.row.to_zero_indexed())? + .nth(loc?.line.to_zero_indexed())? .to_owned(); Some(line + "\n") } diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 9f4add07082..d1821f2e733 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -251,12 +251,12 @@ pub fn syntax_err(err: CompileError) -> SyntaxError { let _ = Reflect::set( &js_err, &"row".into(), - &(err.location().unwrap().row.get()).into(), + &(err.location().unwrap().line.get()).into(), ); let _ = Reflect::set( &js_err, &"col".into(), - &(err.location().unwrap().column.get()).into(), + &(err.location().unwrap().character_offset.get()).into(), ); // | ParseErrorType::UnrecognizedToken(Token::Dedent, _) let can_continue = matches!(