Skip to content

Commit beff2b0

Browse files
widehyo1widehyo1
authored
csv: port CPython reader state machine (#8381)
* csv: port CPython reader state machine Replace the csv-core reader and per-item quote scanner with one Rust implementation of CPython's nine-state reader parser. Keep parser state across iterator items and distinguish virtual item boundaries from true iterator exhaustion. Centralize field completion so quote provenance, empty-field None conversion, float conversion, strict parsing, field limits, blank rows, and escaped or quoted newlines share one path. Apply the existing reentrant-iterator generation check to every iterator item consumed by a record. Also accept empty ASCII line terminators like CPython, report a missing csv-core writer sentinel as _csv.Error instead of panicking, and name the FSM entry point process_parser_input. Remove the expected-failure markers from the eleven reader tests that now pass. Assisted-by: Codex:gpt-5.6-sol * csv: test CRLF boundaries with custom lineterminator
1 parent 4d890b3 commit beff2b0

4 files changed

Lines changed: 311 additions & 375 deletions

File tree

‎.cspell.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
"alnum",
6060
"csock",
6161
"coro",
62+
"Crnl",
6263
"dedentations",
6364
"dedents",
6465
"deduped",

‎Lib/test/test_csv.py‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -371,7 +371,6 @@ def _read_test(self, input, expect, **kwargs):
371371
result = list(reader)
372372
self.assertEqual(result, expect)
373373

374-
@unittest.expectedFailure # TODO: RUSTPYTHON
375374
def test_read_oddinputs(self):
376375
self._read_test([], [])
377376
self._read_test([''], [[]])
@@ -382,7 +381,6 @@ def test_read_oddinputs(self):
382381
self.assertRaises(csv.Error, self._read_test,
383382
[b'abc'], None)
384383

385-
@unittest.expectedFailure # TODO: RUSTPYTHON
386384
def test_read_eol(self):
387385
self._read_test(['a,b', 'c,d'], [['a','b'], ['c','d']])
388386
self._read_test(['a,b\n', 'c,d\n'], [['a','b'], ['c','d']])
@@ -397,7 +395,6 @@ def test_read_eol(self):
397395
with self.assertRaisesRegex(csv.Error, errmsg):
398396
next(csv.reader(['a,b\r\nc,d']))
399397

400-
@unittest.expectedFailure # TODO: RUSTPYTHON
401398
def test_read_eof(self):
402399
self._read_test(['a,"'], [['a', '']])
403400
self._read_test(['"a'], [['a']])
@@ -407,7 +404,6 @@ def test_read_eof(self):
407404
self.assertRaises(csv.Error, self._read_test,
408405
['^'], [], escapechar='^', strict=True)
409406

410-
@unittest.expectedFailure # TODO: RUSTPYTHON
411407
def test_read_nul(self):
412408
self._read_test(['\0'], [['\0']])
413409
self._read_test(['a,\0b,c'], [['a', '\0b', 'c']])
@@ -420,7 +416,6 @@ def test_read_delimiter(self):
420416
self._read_test(['a;b;c'], [['a', 'b', 'c']], delimiter=';')
421417
self._read_test(['a\0b\0c'], [['a', 'b', 'c']], delimiter='\0')
422418

423-
@unittest.expectedFailure # TODO: RUSTPYTHON
424419
def test_read_escape(self):
425420
self._read_test(['a,\\b,c'], [['a', 'b', 'c']], escapechar='\\')
426421
self._read_test(['a,b\\,c'], [['a', 'b,c']], escapechar='\\')
@@ -433,7 +428,6 @@ def test_read_escape(self):
433428
self._read_test(['a,\\b,c'], [['a', '\\b', 'c']], escapechar=None)
434429
self._read_test(['a,\\b,c'], [['a', '\\b', 'c']])
435430

436-
@unittest.expectedFailure # TODO: RUSTPYTHON
437431
def test_read_quoting(self):
438432
self._read_test(['1,",3,",5'], [['1', ',3,', '5']])
439433
self._read_test(['1,",3,",5'], [['1', '"', '3', '"', '5']],
@@ -484,7 +478,6 @@ def test_read_skipinitialspace(self):
484478
[[None, None, None]],
485479
skipinitialspace=True, quoting=csv.QUOTE_STRINGS)
486480

487-
@unittest.expectedFailure # TODO: RUSTPYTHON
488481
def test_read_space_delimiter(self):
489482
self._read_test(['a b', ' a ', ' ', ''],
490483
[['a', '', '', 'b'], ['', '', 'a', '', ''], ['', '', ''], []],
@@ -524,7 +517,6 @@ def test_read_linenum(self):
524517
self.assertRaises(StopIteration, next, r)
525518
self.assertEqual(r.line_num, 3)
526519

527-
@unittest.expectedFailure # TODO: RUSTPYTHON
528520
def test_roundtrip_quoteed_newlines(self):
529521
rows = [
530522
['\na', 'b\nc', 'd\n'],
@@ -543,7 +535,6 @@ def test_roundtrip_quoteed_newlines(self):
543535
for i, row in enumerate(csv.reader(fileobj)):
544536
self.assertEqual(row, rows[i])
545537

546-
@unittest.expectedFailure # TODO: RUSTPYTHON
547538
def test_roundtrip_escaped_unquoted_newlines(self):
548539
rows = [
549540
['\na', 'b\nc', 'd\n'],
@@ -807,7 +798,6 @@ def test_quoted_quote(self):
807798
'"I see," said the blind man',
808799
'as he picked up his hammer and saw']])
809800

810-
@unittest.expectedFailure # TODO: RUSTPYTHON
811801
def test_quoted_nl(self):
812802
input = '''\
813803
1,2,3,"""I see,""
@@ -1078,7 +1068,6 @@ def test_read_multi(self):
10781068
"s1": 'abc',
10791069
"s2": 'def'})
10801070

1081-
@unittest.expectedFailure # TODO: RUSTPYTHON
10821071
def test_read_with_blanks(self):
10831072
reader = csv.DictReader(["1,2,abc,4,5,6\r\n","\r\n",
10841073
"1,2,abc,4,5,6\r\n"],

0 commit comments

Comments
 (0)