Skip to content

Commit 89f29c7

Browse files
miss-islingtonserhiy-storchakaclaude
authored
[3.14] gh-98820: Fix quadratic time in csv.Sniffer for quoted fields (GH-154867) (GH-155117)
The regular expressions which look for a quoted field matched its body lazily, so a closing quote which was not followed by a delimiter was retried with every following quote, to the end of the sample. Match the body possessively instead: it ends at the first quote which is not doubled, as it does for a reader. (cherry picked from commit 476fb09) Co-authored-by: Serhiy Storchaka <storchaka@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent 12ac1ed commit 89f29c7

3 files changed

Lines changed: 18 additions & 5 deletions

File tree

Lib/csv.py

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -288,12 +288,16 @@ def _guess_quote_and_delimiter(self, data, delimiters):
288288
"""
289289
import re
290290

291+
# The body of a quoted field ends at the first quote which is
292+
# not doubled, as it does for a reader. A lazy ".*?" scans to
293+
# the end of the sample instead, from every start: quadratically.
294+
body = r'(?:(?P=quote){2}|(?!(?P=quote)).)*+'
291295
matches = []
292-
for restr in (r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?P=delim)', # ,".*?",
293-
r'(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)', # ".*?",
294-
r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?:$|\r|\n)', # ,".*?"
295-
r'(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?:$|\r|\n)'): # ".*?" (no delim, no space)
296-
regexp = re.compile(restr, re.DOTALL | re.MULTILINE)
296+
for restr in (r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\'])%s(?P=quote)(?P=delim)', # ,"...",
297+
r'(?:^|\n)(?P<quote>["\'])%s(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)', # "...",
298+
r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\'])%s(?P=quote)(?:$|\n)', # ,"..."
299+
r'(?:^|\n)(?P<quote>["\'])%s(?P=quote)(?:$|\n)'): # "..." (no delim, no space)
300+
regexp = re.compile(restr % body, re.DOTALL | re.MULTILINE)
297301
matches = regexp.findall(data)
298302
if matches:
299303
break

Lib/test/test_csv.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1544,6 +1544,13 @@ def test_sniff_space_delimiter(self):
15441544
self.assertEqual(dialect.delimiter, ' ')
15451545
self.assertIs(dialect.doublequote, False)
15461546

1547+
def test_sniff_quoted_single_column(self):
1548+
# gh-98820: this sample used to take minutes.
1549+
sniffer = csv.Sniffer()
1550+
sample = '"abcdefghijklmnopqrstuvwxyz"\n' * 10000
1551+
with self.assertRaisesRegex(csv.Error, "Could not determine delimiter"):
1552+
sniffer.sniff(sample, delimiters=',:|\t')
1553+
15471554

15481555
class NUL:
15491556
def write(s, *args):
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix quadratic time in :meth:`csv.Sniffer.sniff` for a sample which contains
2+
quoted fields, in particular for a single column of quoted fields.

0 commit comments

Comments
 (0)