Skip to content

Commit e7e4d1d

Browse files
chestnut1717chestnut1717
committed
Fix EOF SyntaxError diagnostics
1 parent aa4f98a commit e7e4d1d

6 files changed

Lines changed: 61 additions & 16 deletions

File tree

‎Lib/test/test_eof.py‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ def test_EOF_single_quote(self):
1818
self.assertEqual(str(cm.exception), expect)
1919
self.assertEqual(cm.exception.offset, 1)
2020

21-
@unittest.expectedFailure # TODO: RUSTPYTHON
2221
def test_EOFS(self):
2322
expect = ("unterminated triple-quoted string literal (detected at line 3) (<string>, line 1)")
2423
with self.assertRaises(SyntaxError) as cm:
@@ -45,7 +44,6 @@ def test_EOFS(self):
4544
self.assertEqual(cm.exception.text, "ä = '''thîs is ")
4645
self.assertEqual(cm.exception.offset, 5)
4746

48-
@unittest.expectedFailure # TODO: RUSTPYTHON
4947
@force_not_colorized
5048
def test_EOFS_with_file(self):
5149
expect = ("(<string>, line 1)")
@@ -86,15 +84,13 @@ def test_EOFS_with_file(self):
8684
' ^',
8785
'SyntaxError: unterminated triple-quoted string literal (detected at line 4)'])
8886

89-
@unittest.expectedFailure # TODO: RUSTPYTHON
9087
@warnings_helper.ignore_warnings(category=SyntaxWarning)
9188
def test_eof_with_line_continuation(self):
9289
expect = "unexpected EOF while parsing (<string>, line 1)"
9390
with self.assertRaises(SyntaxError) as cm:
9491
compile('"\\Xhh" \\', '<string>', 'exec')
9592
self.assertEqual(str(cm.exception), expect)
9693

97-
@unittest.expectedFailure # TODO: RUSTPYTHON
9894
def test_line_continuation_EOF(self):
9995
"""A continuation at the end of input must be an error; bpo2180."""
10096
expect = 'unexpected EOF while parsing (<string>, line 1)'
@@ -127,7 +123,6 @@ def test_line_continuation_EOF(self):
127123
exec('\\')
128124
self.assertEqual(str(cm.exception), expect)
129125

130-
@unittest.expectedFailure # TODO: RUSTPYTHON
131126
@unittest.skipIf(not sys.executable, "sys.executable required")
132127
@force_not_colorized
133128
def test_line_continuation_EOF_from_file_bpo2180(self):

‎crates/compiler/src/lib.rs‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -124,9 +124,10 @@ impl CompileError {
124124
}
125125

126126
fn source_location(source_file: &SourceFile, offset: TextSize) -> SourceLocation {
127+
// Python reports SyntaxError columns in Unicode code points.
127128
source_file
128129
.to_source_code()
129-
.source_location(offset, PositionEncoding::Utf8)
130+
.source_location(offset, PositionEncoding::Utf32)
130131
}
131132

132133
fn source_locations(
@@ -136,8 +137,8 @@ fn source_locations(
136137
) -> (SourceLocation, SourceLocation) {
137138
let source_code = source_file.to_source_code();
138139
(
139-
source_code.source_location(start, PositionEncoding::Utf8),
140-
source_code.source_location(end, PositionEncoding::Utf8),
140+
source_code.source_location(start, PositionEncoding::Utf32),
141+
source_code.source_location(end, PositionEncoding::Utf32),
141142
)
142143
}
143144

@@ -223,6 +224,16 @@ fn cpython_parse_diagnostic_override(
223224
&error.error,
224225
parser::ParseErrorType::Lexical(parser::LexicalErrorType::LineContinuationError)
225226
) {
227+
// Only a backslash at the end of the source is an EOF error.
228+
let terminal_backslash = source_text.len().checked_sub(1);
229+
if terminal_backslash == Some(error.location.start().to_usize()) {
230+
let loc = source_line_end_location(source_file, error.location.start());
231+
return Some(NormalizedParseDiagnostic::new(
232+
parser::ParseErrorType::OtherError("unexpected EOF while parsing".to_owned()),
233+
loc,
234+
loc,
235+
));
236+
}
226237
let loc = source_location(source_file, error.location.start() + TextSize::from(1));
227238
return Some(NormalizedParseDiagnostic::new(
228239
error.error.clone(),

‎crates/vm/src/exceptions.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,11 @@ impl VirtualMachine {
245245
_ => true,
246246
};
247247

248-
if same_line {
248+
// A lone continuation at EOF has no highlighted source span.
249+
let lone_line_continuation =
250+
maybe_end_offset == Some(-1) && l_text.to_string_lossy() == "\\";
251+
252+
if same_line && !lone_line_continuation {
249253
let mut end_offset = match maybe_end_offset {
250254
Some(0) | None => offset,
251255
Some(end_offset) => end_offset,

‎crates/vm/src/stdlib/sys.rs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -818,8 +818,27 @@ pub mod sys {
818818
vm: &VirtualMachine,
819819
) -> PyResult<()> {
820820
let stderr = super::get_stderr(vm)?;
821+
// Keep runtime SyntaxErrors on the normal traceback path.
822+
let has_traceback = !vm.is_none(&exc_tb);
821823
match vm.normalize_exception(exc_type, exc_val.clone(), exc_tb) {
822824
Ok(exc) => {
825+
let native_syntax_error_display = !has_traceback
826+
&& exc.fast_isinstance(vm.ctx.exceptions.syntax_error)
827+
&& exc
828+
.as_object()
829+
.get_attr("msg", vm)
830+
.ok()
831+
.and_then(|msg| msg.downcast::<PyStr>().ok())
832+
.is_some_and(|msg| msg.to_string_lossy() == "unexpected EOF while parsing")
833+
&& exc
834+
.as_object()
835+
.get_attr("text", vm)
836+
.ok()
837+
.and_then(|text| text.downcast::<PyStr>().ok())
838+
.is_some_and(|text| text.to_string_lossy().trim_end() == "\\");
839+
if native_syntax_error_display {
840+
return vm.write_exception(&mut crate::py_io::PyWriter(stderr, vm), &exc);
841+
}
823842
// PyErr_Display: try traceback._print_exception_bltin first
824843
if let Ok(tb_mod) = vm.import("traceback", 0)
825844
&& let Ok(print_exc_builtin) = tb_mod.get_attr("_print_exception_bltin", vm)

‎crates/vm/src/vm/python_run.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,10 @@ mod file_run {
113113
"source code cannot contain null bytes".into(),
114114
));
115115
}
116+
#[cfg(feature = "parser")]
117+
// Match compile() by honoring BOMs and encoding cookies in files.
118+
let source = self.decode_source_bytes(&source_bytes, path, false)?;
119+
#[cfg(not(feature = "parser"))]
116120
let source = String::from_utf8(source_bytes)
117121
.map_err(|err| self.new_os_error(err.to_string()))?;
118122
let code_obj = self

‎crates/vm/src/vm/vm_new.rs‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -752,7 +752,7 @@ impl VirtualMachine {
752752
Some(line + "\n")
753753
}
754754

755-
let statement = source.and_then(|src| get_statement(src, error.location()));
755+
let mut statement = source.and_then(|src| get_statement(src, error.location()));
756756

757757
let mut msg = error.to_string();
758758
if !msg.starts_with("Exceeds the limit ")
@@ -799,6 +799,16 @@ impl VirtualMachine {
799799
}
800800

801801
let SyntaxErrorInfo { msg, narrow_caret } = syntax_error_info;
802+
let unterminated_triple_quoted_string =
803+
msg.starts_with("unterminated triple-quoted string literal");
804+
let unexpected_eof_error = msg == "unexpected EOF while parsing";
805+
if unterminated_triple_quoted_string
806+
&& let Some(statement) = statement.as_mut()
807+
&& statement.ends_with('\n')
808+
{
809+
// CPython omits the parser-added final newline from SyntaxError.text.
810+
statement.pop();
811+
}
802812
let check_version_suite_error = msg.starts_with("Async functions are")
803813
|| msg.starts_with("Async for loops are")
804814
|| msg.starts_with("Async with statements are")
@@ -820,12 +830,14 @@ impl VirtualMachine {
820830

821831
// Set end_lineno and end_offset if available
822832
if let Some((end_lineno, end_offset)) = error.python_end_location() {
823-
let (end_lineno, end_offset) = if check_version_suite_error
824-
&& statement
825-
.as_deref()
826-
.and_then(|line| line.chars().next())
827-
.is_some_and(|ch| ch.is_ascii_whitespace())
828-
{
833+
// EOF errors have no source span in CPython.
834+
let no_end_offset = unexpected_eof_error
835+
|| (check_version_suite_error
836+
&& statement
837+
.as_deref()
838+
.and_then(|line| line.chars().next())
839+
.is_some_and(|ch| ch.is_ascii_whitespace()));
840+
let (end_lineno, end_offset) = if no_end_offset {
829841
(end_lineno, -1)
830842
} else if line_end_binary_operator_error && end_offset == offset_raw {
831843
(end_lineno, (end_offset + 1) as isize)

0 commit comments

Comments
 (0)