diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 494cce50a2a..6e00f8eb1d4 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -857,7 +857,6 @@ class TestEscapedExcel(TestCsvBase): def test_escape_fieldsep(self): self.writerAssertEqual([['abc,def']], 'abc\\,def\r\n') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_escape_fieldsep(self): self.readerAssertEqual('abc\\,def\r\n', [['abc,def']]) @@ -881,7 +880,6 @@ class TestQuotedEscapedExcel(TestCsvBase): def test_write_escape_fieldsep(self): self.writerAssertEqual([['abc,def']], '"abc,def"\r\n') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_escape_fieldsep(self): self.readerAssertEqual('"abc\\,def"\r\n', [['abc,def']]) diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs index aaffab18252..48ad68d43ac 100644 --- a/crates/stdlib/src/csv.rs +++ b/crates/stdlib/src/csv.rs @@ -797,45 +797,27 @@ mod _csv { } fn to_reader(&self) -> csv_core::Reader { - let mut builder = csv_core::ReaderBuilder::new(); - let mut reader = match &self.dialect { - DialectItem::Str(name) => { + let dialect = match &self.dialect { + DialectItem::Str(name) => GLOBAL_HASHMAP.lock().get(name).copied(), + DialectItem::Obj(obj) => Some(*obj), + DialectItem::None => { let g = GLOBAL_HASHMAP.lock(); - if let Some(dialect) = g.get(name) { - let mut builder = builder - .delimiter(dialect.delimiter) - .double_quote(dialect.doublequote); - if let Some(t) = dialect.quotechar { - builder = builder.quote(t); - } - builder - // RustPython todo - // todo! Perfecting the remaining attributes. - } else { - &mut builder - } - } - DialectItem::Obj(obj) => { - let mut builder = builder - .delimiter(obj.delimiter) - .double_quote(obj.doublequote); - if let Some(t) = obj.quotechar { - builder = builder.quote(t); - } - builder + Some(*g.get("excel").unwrap()) } - _ => { - let name = "excel"; - let g = GLOBAL_HASHMAP.lock(); - let dialect = g.get(name).unwrap(); - let mut builder = builder - .delimiter(dialect.delimiter) - .double_quote(dialect.doublequote); - if let Some(quotechar) = dialect.quotechar { - builder = builder.quote(quotechar); - } - builder + }; + + let mut builder = csv_core::ReaderBuilder::new(); + let mut reader = if let Some(dialect) = dialect { + let mut builder = builder + .delimiter(dialect.delimiter) + .double_quote(dialect.doublequote) + .escape(dialect.escapechar); + if let Some(quotechar) = dialect.quotechar { + builder = builder.quote(quotechar); } + builder + } else { + &mut builder }; if let Some(t) = self.delimiter { @@ -972,6 +954,67 @@ mod _csv { impl SelfIter for Reader {} + fn read_quote_none_record( + input: &[u8], + dialect: PyDialect, + field_limit: isize, + vm: &VirtualMachine, + ) -> PyResult> { + let mut fields = vec![Vec::new()]; + let mut escaped = false; + let mut after_delimiter = false; + + for (index, &byte) in input.iter().enumerate() { + if escaped { + fields.last_mut().unwrap().push(byte); + escaped = false; + after_delimiter = false; + } else if dialect.skipinitialspace && after_delimiter && byte == b' ' { + continue; + } else if dialect.escapechar == Some(byte) { + escaped = true; + } else if byte == dialect.delimiter { + fields.push(Vec::new()); + after_delimiter = true; + } else if matches!(byte, b'\r' | b'\n') { + if !input[index..] + .iter() + .all(|&byte| matches!(byte, b'\r' | b'\n')) + { + return Err(new_csv_error( + vm, + concat!( + "new-line character seen in unquoted field", + " - do you need to open the file in universal-newline mode?" + ), + )); + } + break; + } else { + fields.last_mut().unwrap().push(byte); + after_delimiter = false; + } + } + + // CPython treats an escape character at the end of an iterator item + // as escaping the implicit newline at the end of that item. + if escaped { + fields.last_mut().unwrap().push(b'\n'); + } + + fields + .into_iter() + .map(|field| { + if field.len() > field_limit as usize { + return Err(new_csv_error(vm, "filed too long to read")); + } + let field = core::str::from_utf8(&field) + .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?; + Ok(vm.ctx.new_str(field).into()) + }) + .collect() + } + impl IterNext for Reader { fn next(zelf: &Py, vm: &VirtualMachine) -> PyResult { let string = raise_if_stop!(zelf.iter.next(vm)?); @@ -1003,6 +1046,12 @@ mod _csv { let mut output_ends_offset = 0; let field_limit = GLOBAL_FIELD_LIMIT.lock().to_owned(); + if zelf.dialect.quoting == QuoteStyle::None && zelf.dialect.escapechar.is_some() { + let out = read_quote_none_record(input, zelf.dialect, field_limit, vm)?; + *line_num += 1; + return Ok(PyIterReturn::Return(vm.ctx.new_list(out).into())); + } + #[inline] fn trim_spaces(input: &[u8]) -> &[u8] { let trimmed_start = input.iter().position(|&x| x != b' ').unwrap_or(input.len()); diff --git a/extra_tests/snippets/stdlib_csv.py b/extra_tests/snippets/stdlib_csv.py index b9c741cbb16..dc2186d17ac 100644 --- a/extra_tests/snippets/stdlib_csv.py +++ b/extra_tests/snippets/stdlib_csv.py @@ -134,3 +134,16 @@ def test_quote_none_writer_without_quotechar(): test_quote_none_writer_without_quotechar() + + +def test_quote_none_reader_skipinitialspace_escapechar(): + reader = csv.reader( + ["a, b,\\ c,d"], + quoting=csv.QUOTE_NONE, + escapechar="\\", + skipinitialspace=True, + ) + assert list(reader) == [["a", "b", " c", "d"]] + + +test_quote_none_reader_skipinitialspace_escapechar()