From 600eb446531a1c27cee3d455af87655daafc2bba Mon Sep 17 00:00:00 2001 From: "widehyo@gmail.com" Date: Sun, 12 Jul 2026 16:51:33 +0900 Subject: [PATCH 1/4] csv: apply dialect escapechar when reading ReaderBuilder only received escapechar when it was passed directly as a keyword argument. A dialect object's or registered dialect's escapechar was visible through reader.dialect but not used by csv_core. Pass escapechar through for named, object, and default dialect paths. This makes escaped delimiters inside quoted fields follow the configured dialect, for example '"abc\,def"' reads as 'abc,def'. Unmark TestQuotedEscapedExcel.test_read_escape_fieldsep. Assisted-by: Codex --- Lib/test/test_csv.py | 1 - crates/stdlib/src/csv.rs | 9 ++++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 494cce50a2a..63535584d7d 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -881,7 +881,6 @@ class TestQuotedEscapedExcel(TestCsvBase): def test_write_escape_fieldsep(self): self.writerAssertEqual([['abc,def']], '"abc,def"\r\n') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_escape_fieldsep(self): self.readerAssertEqual('"abc\\,def"\r\n', [['abc,def']]) diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs index aaffab18252..1151823e105 100644 --- a/crates/stdlib/src/csv.rs +++ b/crates/stdlib/src/csv.rs @@ -804,7 +804,8 @@ mod _csv { if let Some(dialect) = g.get(name) { let mut builder = builder .delimiter(dialect.delimiter) - .double_quote(dialect.doublequote); + .double_quote(dialect.doublequote) + .escape(dialect.escapechar); if let Some(t) = dialect.quotechar { builder = builder.quote(t); } @@ -818,7 +819,8 @@ mod _csv { DialectItem::Obj(obj) => { let mut builder = builder .delimiter(obj.delimiter) - .double_quote(obj.doublequote); + .double_quote(obj.doublequote) + .escape(obj.escapechar); if let Some(t) = obj.quotechar { builder = builder.quote(t); } @@ -830,7 +832,8 @@ mod _csv { let dialect = g.get(name).unwrap(); let mut builder = builder .delimiter(dialect.delimiter) - .double_quote(dialect.doublequote); + .double_quote(dialect.doublequote) + .escape(dialect.escapechar); if let Some(quotechar) = dialect.quotechar { builder = builder.quote(quotechar); } From 562aba0ddfb8ee089ce3c5b11934b3ee9f842cad Mon Sep 17 00:00:00 2001 From: "widehyo@gmail.com" Date: Sun, 12 Jul 2026 17:01:40 +0900 Subject: [PATCH 2/4] csv: handle escaped delimiters with QUOTE_NONE csv_core applies escapechar while reading quoted fields, but its unquoted field state treats an escaped delimiter as a delimiter. Consequently, QUOTE_NONE with escapechar split 'abc\,def' into 'abc\' and 'def'. Add a small QUOTE_NONE record parser for dialects with escapechar. It keeps the byte following an escape character as field data, so escaped delimiters do not end a field while ordinary delimiters and record terminators retain their usual meaning. Unmark TestEscapedExcel.test_read_escape_fieldsep. Assisted-by: Codex --- Lib/test/test_csv.py | 1 - crates/stdlib/src/csv.rs | 61 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 63535584d7d..6e00f8eb1d4 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -857,7 +857,6 @@ class TestEscapedExcel(TestCsvBase): def test_escape_fieldsep(self): self.writerAssertEqual([['abc,def']], 'abc\\,def\r\n') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_escape_fieldsep(self): self.readerAssertEqual('abc\\,def\r\n', [['abc,def']]) diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs index 1151823e105..76e6864ec0e 100644 --- a/crates/stdlib/src/csv.rs +++ b/crates/stdlib/src/csv.rs @@ -975,6 +975,61 @@ mod _csv { impl SelfIter for Reader {} + fn read_quote_none_record( + input: &[u8], + dialect: PyDialect, + field_limit: isize, + vm: &VirtualMachine, + ) -> PyResult> { + let mut fields = vec![Vec::new()]; + let mut escaped = false; + + for (index, &byte) in input.iter().enumerate() { + if escaped { + fields.last_mut().unwrap().push(byte); + escaped = false; + } else if dialect.escapechar == Some(byte) { + escaped = true; + } else if byte == dialect.delimiter { + fields.push(Vec::new()); + } else if matches!(byte, b'\r' | b'\n') { + if !input[index..] + .iter() + .all(|&byte| matches!(byte, b'\r' | b'\n')) + { + return Err(new_csv_error( + vm, + concat!( + "new-line character seen in unquoted field", + " - do you need to open the file in universal-newline mode?" + ), + )); + } + break; + } else { + fields.last_mut().unwrap().push(byte); + } + } + + // CPython treats an escape character at the end of an iterator item + // as escaping the implicit newline at the end of that item. + if escaped { + fields.last_mut().unwrap().push(b'\n'); + } + + fields + .into_iter() + .map(|field| { + if field.len() > field_limit as usize { + return Err(new_csv_error(vm, "filed too long to read")); + } + let field = core::str::from_utf8(&field) + .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?; + Ok(vm.ctx.new_str(field).into()) + }) + .collect() + } + impl IterNext for Reader { fn next(zelf: &Py, vm: &VirtualMachine) -> PyResult { let string = raise_if_stop!(zelf.iter.next(vm)?); @@ -1006,6 +1061,12 @@ mod _csv { let mut output_ends_offset = 0; let field_limit = GLOBAL_FIELD_LIMIT.lock().to_owned(); + if zelf.dialect.quoting == QuoteStyle::None && zelf.dialect.escapechar.is_some() { + let out = read_quote_none_record(input, zelf.dialect, field_limit, vm)?; + *line_num += 1; + return Ok(PyIterReturn::Return(vm.ctx.new_list(out).into())); + } + #[inline] fn trim_spaces(input: &[u8]) -> &[u8] { let trimmed_start = input.iter().position(|&x| x != b' ').unwrap_or(input.len()); From f9105a4645fad260c058ab04868a319c3f3c5646 Mon Sep 17 00:00:00 2001 From: "widehyo@gmail.com" Date: Sun, 12 Jul 2026 20:03:43 +0900 Subject: [PATCH 3/4] csv: respect skipinitialspace with QUOTE_NONE The QUOTE_NONE parser returned before the normal reader's whitespace preprocessing, so it ignored skipinitialspace when escapechar was set. Track whether parsing has just followed a delimiter and skip only ordinary spaces in that position. Escaped spaces remain field data. Add a stdlib_csv snippet regression test for QUOTE_NONE with escapechar and skipinitialspace. Assisted-by: Codex:gpt-5.6-terra --- crates/stdlib/src/csv.rs | 6 ++++++ extra_tests/snippets/stdlib_csv.py | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs index 76e6864ec0e..f019ec031b7 100644 --- a/crates/stdlib/src/csv.rs +++ b/crates/stdlib/src/csv.rs @@ -983,15 +983,20 @@ mod _csv { ) -> PyResult> { let mut fields = vec![Vec::new()]; let mut escaped = false; + let mut after_delimiter = false; for (index, &byte) in input.iter().enumerate() { if escaped { fields.last_mut().unwrap().push(byte); escaped = false; + after_delimiter = false; + } else if dialect.skipinitialspace && after_delimiter && byte == b' ' { + continue; } else if dialect.escapechar == Some(byte) { escaped = true; } else if byte == dialect.delimiter { fields.push(Vec::new()); + after_delimiter = true; } else if matches!(byte, b'\r' | b'\n') { if !input[index..] .iter() @@ -1008,6 +1013,7 @@ mod _csv { break; } else { fields.last_mut().unwrap().push(byte); + after_delimiter = false; } } diff --git a/extra_tests/snippets/stdlib_csv.py b/extra_tests/snippets/stdlib_csv.py index b9c741cbb16..dc2186d17ac 100644 --- a/extra_tests/snippets/stdlib_csv.py +++ b/extra_tests/snippets/stdlib_csv.py @@ -134,3 +134,16 @@ def test_quote_none_writer_without_quotechar(): test_quote_none_writer_without_quotechar() + + +def test_quote_none_reader_skipinitialspace_escapechar(): + reader = csv.reader( + ["a, b,\\ c,d"], + quoting=csv.QUOTE_NONE, + escapechar="\\", + skipinitialspace=True, + ) + assert list(reader) == [["a", "b", " c", "d"]] + + +test_quote_none_reader_skipinitialspace_escapechar() From 0ef9af95b146a6399f12c4a560f7301f54512f72 Mon Sep 17 00:00:00 2001 From: "widehyo@gmail.com" Date: Sun, 12 Jul 2026 20:34:49 +0900 Subject: [PATCH 4/4] csv: deduplicate reader dialect configuration to_reader configured ReaderBuilder separately for named, object, and default dialects even though the configuration was identical. Resolve the PyDialect once, then apply delimiter, double_quote, escapechar, and quotechar in one place. Preserve the existing unregistered-name fallback and required excel dialect lookup behavior. Assisted-by: Codex:gpt-5.6-terra --- crates/stdlib/src/csv.rs | 57 +++++++++++++--------------------------- 1 file changed, 18 insertions(+), 39 deletions(-) diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs index f019ec031b7..48ad68d43ac 100644 --- a/crates/stdlib/src/csv.rs +++ b/crates/stdlib/src/csv.rs @@ -797,48 +797,27 @@ mod _csv { } fn to_reader(&self) -> csv_core::Reader { - let mut builder = csv_core::ReaderBuilder::new(); - let mut reader = match &self.dialect { - DialectItem::Str(name) => { + let dialect = match &self.dialect { + DialectItem::Str(name) => GLOBAL_HASHMAP.lock().get(name).copied(), + DialectItem::Obj(obj) => Some(*obj), + DialectItem::None => { let g = GLOBAL_HASHMAP.lock(); - if let Some(dialect) = g.get(name) { - let mut builder = builder - .delimiter(dialect.delimiter) - .double_quote(dialect.doublequote) - .escape(dialect.escapechar); - if let Some(t) = dialect.quotechar { - builder = builder.quote(t); - } - builder - // RustPython todo - // todo! Perfecting the remaining attributes. - } else { - &mut builder - } + Some(*g.get("excel").unwrap()) } - DialectItem::Obj(obj) => { - let mut builder = builder - .delimiter(obj.delimiter) - .double_quote(obj.doublequote) - .escape(obj.escapechar); - if let Some(t) = obj.quotechar { - builder = builder.quote(t); - } - builder - } - _ => { - let name = "excel"; - let g = GLOBAL_HASHMAP.lock(); - let dialect = g.get(name).unwrap(); - let mut builder = builder - .delimiter(dialect.delimiter) - .double_quote(dialect.doublequote) - .escape(dialect.escapechar); - if let Some(quotechar) = dialect.quotechar { - builder = builder.quote(quotechar); - } - builder + }; + + let mut builder = csv_core::ReaderBuilder::new(); + let mut reader = if let Some(dialect) = dialect { + let mut builder = builder + .delimiter(dialect.delimiter) + .double_quote(dialect.doublequote) + .escape(dialect.escapechar); + if let Some(quotechar) = dialect.quotechar { + builder = builder.quote(quotechar); } + builder + } else { + &mut builder }; if let Some(t) = self.delimiter {