Skip to content

Commit 0c75438

Browse files
widehyo1youknowone
widehyo1
authored and
youknowone
committed
csv: handle empty fields with skipinitialspace
skipinitialspace preprocessed records by trimming both ends of every split field. Fields containing only spaces could therefore produce an invalid slice, while trailing spaces were removed even though CPython preserves them. Trim only field prefixes in the csv-core path and make the remaining all-space trimming helper return an empty slice safely. QUOTE_NOTNULL and QUOTE_STRINGS add another distinction: an unquoted empty field becomes None, but a quoted empty field remains an empty string. Both forms have identical decoded bytes, so extend the existing QUOTE_NONE custom reader into one shared path that records whether each field started quoted. Use that metadata only for the null conversion while retaining QUOTE_NONE escape behavior. Unmark Test_Csv.test_read_skipinitialspace now that its standard, QUOTE_NOTNULL, and QUOTE_STRINGS cases pass. This keeps the existing one-item reader lifecycle and does not add the larger strict or multiline parser state machine. Assisted-by: Codex:gpt-5-sol
1 parent 8754004 commit 0c75438

2 files changed

Lines changed: 80 additions & 19 deletions

File tree

‎Lib/test/test_csv.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -471,7 +471,6 @@ def test_read_quoting(self):
471471
self._read_test(['1\\.5,\\.5,"\\.5"'], [[1.5, 0.5, ".5"]],
472472
quoting=csv.QUOTE_STRINGS, escapechar='\\')
473473

474-
@unittest.skip("TODO: RUSTPYTHON; slice index starts at 1 but ends at 0")
475474
def test_read_skipinitialspace(self):
476475
self._read_test(['no space, space, spaces,\ttab'],
477476
[['no space', 'space', 'spaces', '\ttab']],

‎crates/stdlib/src/csv.rs‎

Lines changed: 80 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -994,28 +994,68 @@ mod _csv {
994994

995995
impl SelfIter for Reader {}
996996

997-
fn read_quote_none_record(
997+
fn read_quote_record(
998998
input: &[u8],
999999
dialect: PyDialect,
10001000
field_limit: isize,
10011001
vm: &VirtualMachine,
10021002
) -> PyResult<Vec<PyObjectRef>> {
1003-
let mut fields = vec![Vec::new()];
1003+
// QUOTE_NOTNULL and QUOTE_STRINGS map empty unquoted fields to None,
1004+
// but preserve quoted empty fields as strings, so retain quote provenance.
1005+
let mut fields = vec![(Vec::new(), false)];
1006+
let mut at_field_start = true;
1007+
let mut in_quoted_field = false;
10041008
let mut escaped = false;
1005-
let mut after_delimiter = false;
1009+
let mut index = 0;
1010+
1011+
while index < input.len() {
1012+
let byte = input[index];
10061013

1007-
for (index, &byte) in input.iter().enumerate() {
10081014
if escaped {
1009-
fields.last_mut().unwrap().push(byte);
1015+
fields.last_mut().unwrap().0.push(byte);
10101016
escaped = false;
1011-
after_delimiter = false;
1012-
} else if dialect.skipinitialspace && after_delimiter && byte == b' ' {
1017+
at_field_start = false;
1018+
index += 1;
10131019
continue;
1014-
} else if dialect.escapechar == Some(byte) {
1020+
}
1021+
1022+
if dialect.escapechar == Some(byte) {
10151023
escaped = true;
1024+
at_field_start = false;
1025+
index += 1;
1026+
continue;
1027+
}
1028+
1029+
if in_quoted_field {
1030+
if dialect.quotechar == Some(byte) {
1031+
if dialect.doublequote && input.get(index + 1) == Some(&byte) {
1032+
fields.last_mut().unwrap().0.push(byte);
1033+
index += 2;
1034+
} else {
1035+
in_quoted_field = false;
1036+
index += 1;
1037+
}
1038+
} else {
1039+
fields.last_mut().unwrap().0.push(byte);
1040+
index += 1;
1041+
}
1042+
continue;
1043+
}
1044+
1045+
if at_field_start && dialect.skipinitialspace && byte == b' ' {
1046+
index += 1;
1047+
} else if at_field_start
1048+
&& dialect.quoting != QuoteStyle::None
1049+
&& dialect.quotechar == Some(byte)
1050+
{
1051+
fields.last_mut().unwrap().1 = true;
1052+
at_field_start = false;
1053+
in_quoted_field = true;
1054+
index += 1;
10161055
} else if byte == dialect.delimiter {
1017-
fields.push(Vec::new());
1018-
after_delimiter = true;
1056+
fields.push((Vec::new(), false));
1057+
at_field_start = true;
1058+
index += 1;
10191059
} else if matches!(byte, b'\r' | b'\n') {
10201060
if !input[index..]
10211061
.iter()
@@ -1031,23 +1071,30 @@ mod _csv {
10311071
}
10321072
break;
10331073
} else {
1034-
fields.last_mut().unwrap().push(byte);
1035-
after_delimiter = false;
1074+
fields.last_mut().unwrap().0.push(byte);
1075+
at_field_start = false;
1076+
index += 1;
10361077
}
10371078
}
10381079

10391080
// CPython treats an escape character at the end of an iterator item
10401081
// as escaping the implicit newline at the end of that item.
10411082
if escaped {
1042-
fields.last_mut().unwrap().push(b'\n');
1083+
fields.last_mut().unwrap().0.push(b'\n');
10431084
}
10441085

10451086
fields
10461087
.into_iter()
1047-
.map(|field| {
1088+
.map(|(field, was_quoted)| {
10481089
if field.len() > field_limit as usize {
10491090
return Err(new_csv_error(vm, "filed too long to read"));
10501091
}
1092+
if matches!(dialect.quoting, QuoteStyle::Notnull | QuoteStyle::Strings)
1093+
&& !was_quoted
1094+
&& field.is_empty()
1095+
{
1096+
return Ok(vm.ctx.none());
1097+
}
10511098
let field = core::str::from_utf8(&field)
10521099
.map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?;
10531100
Ok(vm.ctx.new_str(field).into())
@@ -1086,23 +1133,38 @@ mod _csv {
10861133
let mut output_ends_offset = 0;
10871134
let field_limit = GLOBAL_FIELD_LIMIT.lock().to_owned();
10881135

1089-
if zelf.dialect.quoting == QuoteStyle::None && zelf.dialect.escapechar.is_some() {
1090-
let out = read_quote_none_record(input, zelf.dialect, field_limit, vm)?;
1136+
let use_quote_record = matches!(
1137+
zelf.dialect.quoting,
1138+
QuoteStyle::Notnull | QuoteStyle::Strings
1139+
) || (zelf.dialect.quoting == QuoteStyle::None
1140+
&& zelf.dialect.escapechar.is_some());
1141+
if use_quote_record {
1142+
let out = read_quote_record(input, zelf.dialect, field_limit, vm)?;
10911143
*line_num += 1;
10921144
return Ok(PyIterReturn::Return(vm.ctx.new_list(out).into()));
10931145
}
10941146

1147+
#[inline]
1148+
fn trim_initial_spaces(input: &[u8]) -> &[u8] {
1149+
let trimmed_start = input.iter().position(|&x| x != b' ').unwrap_or(input.len());
1150+
&input[trimmed_start..]
1151+
}
1152+
10951153
#[inline]
10961154
fn trim_spaces(input: &[u8]) -> &[u8] {
10971155
let trimmed_start = input.iter().position(|&x| x != b' ').unwrap_or(input.len());
10981156
let trimmed_end = input.iter().rposition(|&x| x != b' ').map_or(0, |i| i + 1);
1099-
&input[trimmed_start..trimmed_end]
1157+
if trimmed_start >= trimmed_end {
1158+
&input[input.len()..]
1159+
} else {
1160+
&input[trimmed_start..trimmed_end]
1161+
}
11001162
}
11011163

11021164
let input = if *skipinitialspace {
11031165
let t = input.split(|x| x == delimiter);
11041166
t.map(|x| {
1105-
let trimmed = trim_spaces(x);
1167+
let trimmed = trim_initial_spaces(x);
11061168
String::from_utf8(trimmed.to_vec()).unwrap()
11071169
})
11081170
.join(format!("{}", *delimiter as char).as_str())

0 commit comments

Comments
 (0)