From 7c9f890dea995570de5478de5ce5f72b7ee90848 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 14 Jul 2026 11:12:51 +0300 Subject: [PATCH] gh-83273: Rewrite csv.Sniffer dialect detection using trial parsing Guess the dialect by parsing the sample with every plausible combination of delimiter, quotechar and escapechar, using the actual CSV parser in strict mode, and choosing the combination which splits the sample into rows with the most consistent number of fields. The old heuristics, which guessed the delimiter from characters adjacent to quotes and from character frequencies, are removed. A large sample is parsed incrementally: first only its beginning, then, after eliminating the combinations which are clearly worse than the leader, a several times larger part, and so on. * csv.Sniffer can now detect escapechar='\\'. * Explicitly requested delimiters are no longer restricted to ASCII. * A delimiter inside a quoted field no longer wins over the actual delimiter, and sniffing no longer takes quadratic time on quoted samples. * The sample can be cut off at an arbitrary point: in the middle of a row, of a quoted field or of an escaped sequence. * Only '\r', '\n' and '\r\n' are treated as row separators, so characters like '\x1c' can now be detected as a delimiter. * A preamble (title or comment lines) before the data does not prevent detection if the data rows outnumber the preamble lines. * A sample consisting of a single column of quoted fields now raises csv.Error instead of guessing a delimiter from the content of the fields, and Sniffer.has_header() no longer raises ValueError for such samples. * doublequote is detected by comparing the two readings of the sample. Co-Authored-By: Claude Fable 5 --- Doc/library/csv.rst | 16 + Doc/whatsnew/3.16.rst | 15 + Lib/csv.py | 485 +++++++++++------- Lib/test/test_csv.py | 204 +++++++- ...6-07-14-12-00-00.gh-issue-83273.SnifQ7.rst | 10 + 5 files changed, 550 insertions(+), 180 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-14-12-00-00.gh-issue-83273.SnifQ7.rst diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 87c2d4702e7408..36db964a8bbfe4 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -312,6 +312,22 @@ The :mod:`!csv` module defines the following classes: is given, it is interpreted as a string containing possible valid delimiter characters. + The dialect is deduced by parsing the sample with every plausible + combination of parameters + and choosing the combination which splits the sample into rows + with the most consistent number of fields, + so the returned dialect is consistent with how :func:`reader` + will parse the sample. + Raise :exc:`Error` if no combination fits the sample, + in particular if it is a single column, + so there is no delimiter to find. + + .. versionchanged:: next + The dialect is now deduced by trial parsing + and the results may differ from those of earlier Python versions. + The *escapechar* parameter can now be detected, + and the requested *delimiters* are not restricted to ASCII. + .. method:: has_header(sample) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index da98f1ad6b9bc3..98b795de8250d9 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -96,6 +96,21 @@ codecs even when a built-in codec of the same name exists. (Contributed by Serhiy Storchaka in :gh:`152997`.) +csv +--- + +* :meth:`csv.Sniffer.sniff` now deduces the dialect by trial parsing + instead of heuristics based on matching isolated fragments + and on character frequencies, + so the result is consistent with how :func:`csv.reader` will parse the sample. + It can now detect the *escapechar* parameter, + accepts non-ASCII delimiters in the *delimiters* argument, + no longer misdetects the delimiter + when the sample contains delimiter characters inside quoted fields, + and no longer takes quadratic time on quoted samples. + The results may differ from those of earlier Python versions. + (Contributed by Serhiy Storchaka in :gh:`83273`.) + curses ------ diff --git a/Lib/csv.py b/Lib/csv.py index b2aaf5fd9fa91e..505e9e501e6772 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -233,6 +233,11 @@ class Sniffer: "Sniffs" the format of a CSV file (i.e. delimiter, quotechar) Returns a Dialect object. ''' + # Characters which can be guessed as a delimiter if the delimiters + # argument is not specified. + _delimiter_candidates = [c for c in map(chr, range(128)) + if not c.isalnum()] + def __init__(self): # in case there is more than one possible delimiter self.preferred = [',', '\t', ';', ' ', ':'] @@ -240,214 +245,336 @@ def __init__(self): def sniff(self, sample, delimiters=None): """ - Returns a dialect (or None) corresponding to the sample + Analyze the sample and return a Dialect subclass reflecting the + parameters found. If the optional delimiters parameter is + given, it is interpreted as a string containing possible valid + delimiter characters. Raises Error if the dialect cannot be + determined. + + The dialect is guessed by parsing the sample with every + plausible combination of delimiter, quotechar and escapechar, + and choosing the combination which splits the sample into rows + with the most consistent number of fields. + + A large sample is parsed incrementally: first only its + beginning, then, after eliminating the combinations which are + clearly worse than the leader, a several times larger part, + and so on. """ + import re + from collections import defaultdict - quotechar, doublequote, delimiter, skipinitialspace = \ - self._guess_quote_and_delimiter(sample, delimiters) - if not delimiter: - delimiter, skipinitialspace = self._guess_delimiter(sample, - delimiters) + if self._parses_as_single_column(sample): + # There is no delimiter to find; any combination could + # only find a bogus one inside the quoted fields. + raise Error("Could not determine delimiter") - if not delimiter: + chars = set(sample) + if delimiters is None: + delimiters = self._delimiter_candidates + delimiters = [d for d in delimiters + if d in chars and d not in '\r\n"\'\\'] + # Combinations to try, numbered by preference for breaking + # ties. The unquoted combinations are parsed from the start; + # the rest stay dormant until the quote character occurs at + # the start of a field (see _split_dormant). + groups = defaultdict(list) + order = 0 + # Only '\\' is tried as an escape character: others are not + # seen in the wild. + for escapechar in ('', '\\') if '\\' in chars else ('',): + for quotechar in '"', "'", '': + if quotechar and quotechar not in chars: + continue + for delimiter in delimiters: + groups[quotechar].append( + (order, delimiter, quotechar, escapechar)) + order += 1 + active = groups.pop('', []) + # Only non-empty groups were created; a plain dict cannot + # grow one by accident. + dormant = dict(groups) + + # The initial window should cover the minimal number of rows + # required for elimination (see _eliminate_worse) at a typical + # line length, so that the first round can already eliminate. + window = 2000 + # A line with its line break: '\r', '\n' or '\r\n' (the + # reader treats other line boundary characters as ordinary + # data, but does not support a bare '\r' inside a chunk). + # The \z alternative produces one final empty match. + line_re = re.compile(r'[^\r\n]*(?:\r\n|[\r\n]|\z)') + parsed = [] + lines = [] + first_round = True + while active or dormant: + end = min(window, len(sample)) + part = sample[:end] + lines = line_re.findall(part) + del lines[-1] + cut = not part.endswith(('\r', '\n')) + for quotechar in list(dormant): + activated, still = self._split_dormant( + part, quotechar, dormant[quotechar]) + active += activated + if still: + dormant[quotechar] = still + else: + del dormant[quotechar] + parsed = [(combo, self._try_dialect(lines, cut, *combo[1:])) + for combo in active] + if end == len(sample): + break + active = self._eliminate_worse(parsed, not first_round) + first_round = False + if len(active) <= 3: + # Quoted data most often leaves three survivors: the + # true dialect, its equally consistent unquoted shadow, + # and one accident. Parsing the whole sample with them + # is cheaper than another elimination round. + window = len(sample) + else: + # Too small a factor would increase the total + # re-parsing cost, too large -- the cost of the next + # round if this one did not eliminate enough. + window *= 4 + + best = None + best_score = None + for combo, rows in sorted(parsed): + if rows is None: + continue + _, delimiter, quotechar, escapechar = combo + nfields, share = self._modal_share(rows) + if nfields < 2: + # The delimiter does not delimit anything. + continue + try: + preference = -self.preferred.index(delimiter) + except ValueError: + preference = -len(self.preferred) + # A successful quoted parse is direct evidence; the preferred + # delimiters list is only a nudge. + score = (share, len(rows), bool(quotechar), preference) + if best_score is None or score > best_score: + best_score = score + best = combo[1:] + + if best is None: raise Error("Could not determine delimiter") + delimiter, quotechar, escapechar = best + doublequote = self._detect_doublequote(lines, *best) + skipinitialspace = self._detect_skipinitialspace(lines, *best, + doublequote) class dialect(Dialect): _name = "sniffed" lineterminator = '\r\n' quoting = QUOTE_MINIMAL - # escapechar = '' - dialect.doublequote = doublequote dialect.delimiter = delimiter # _csv.reader won't accept a quotechar of '' dialect.quotechar = quotechar or '"' + dialect.escapechar = escapechar or None + dialect.doublequote = doublequote dialect.skipinitialspace = skipinitialspace return dialect - - def _guess_quote_and_delimiter(self, data, delimiters): + def _parses_as_single_column(self, sample): """ - Looks for text enclosed between two identical quotes - (the probable quotechar) which are preceded and followed - by the same character (the probable delimiter). - For example: - ,'some text', - The quote with the most wins, same with the delimiter. - If there is no quotechar the delimiter can't be determined - this way. + True if the whole sample parses as a single column of quoted + fields (the last one may be cut off in the middle), so there + is no delimiter to find. """ import re - matches = [] - for restr in (r'(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?P=delim)', # ,".*?", - r'(?:^|\n)(?P["\']).*?(?P=quote)(?P[^\w\n"\'])(?P ?)', # ".*?", - r'(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?:$|\n)', # ,".*?" - r'(?:^|\n)(?P["\']).*?(?P=quote)(?:$|\n)'): # ".*?" (no delim, no space) - regexp = re.compile(restr, re.DOTALL | re.MULTILINE) - matches = regexp.findall(data) - if matches: - break - - if not matches: - # (quotechar, doublequote, delimiter, skipinitialspace) - return ('', False, None, 0) - quotes = {} - delims = {} - spaces = 0 - groupindex = regexp.groupindex - for m in matches: - n = groupindex['quote'] - 1 - key = m[n] - if key: - quotes[key] = quotes.get(key, 0) + 1 - try: - n = groupindex['delim'] - 1 - key = m[n] - except KeyError: - continue - if key and (delimiters is None or key in delimiters): - delims[key] = delims.get(key, 0) + 1 - try: - n = groupindex['space'] - 1 - except KeyError: - continue - if m[n]: - spaces += 1 - - quotechar = max(quotes, key=quotes.get) - - if delims: - delim = max(delims, key=delims.get) - skipinitialspace = delims[delim] == spaces - if delim == '\n': # most likely a file with a single column - delim = '' - else: - # there is *no* delimiter, it's a single column of quoted data - delim = '' - skipinitialspace = 0 - - # if we see an extra quote between delimiters, we've got a - # double quoted format - dq_regexp = re.compile( - r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \ - {'delim':re.escape(delim), 'quote':quotechar}, re.MULTILINE) - + for q in '"', "'": + if q in sample: + row_re = (fr' *+{q}(?:[^{q}]|{q}{q})*+' + fr'(?:{q} *+(?:[\r\n]++|\z)|\z)') + if re.fullmatch(fr'(?:{row_re})++', sample): + return True + return False + def _split_dormant(self, part, quotechar, combos): + """ + Split the dormant combinations into those ready for trial + parsing and the rest. + + A combination is ready when its quote character occurs in + *part* at the start of a field, i.e. at the start of a line or + after its delimiter; until then parsing would not differ from + the unquoted variant. Spaces before the quote are allowed even + for the space delimiter: a false activation only costs a trial + parse. + """ + import re - if dq_regexp.search(data): - doublequote = True + remaining = {combo[1] for combo in combos} + found = set() + pos = 0 + while remaining: + # Include only the delimiters not found yet, so that the + # search skips over the found ones; the compiled patterns + # come from the re cache. + cls = re.escape(''.join(sorted(remaining))) + m = re.compile(fr'(?:^|([\r\n{cls}]))' + fr' *{quotechar}').search(part, pos) + if m is None: + break + pre = m[1] + if pre is None or pre in '\r\n': + # A quote at the start of a line starts a field for + # every delimiter. + found |= remaining + break + found.add(pre) + remaining.discard(pre) + pos = m.end() + activated = [combo for combo in combos if combo[1] in found] + still_dormant = [combo for combo in combos if combo[1] not in found] + return activated, still_dormant + + def _make_reader(self, lines, delimiter, quotechar, escapechar, + doublequote=True, skipinitialspace=None): + """ + Create a reader for trial parsing. quotechar '' means no + quoting and escapechar '' means no escape character. + """ + if skipinitialspace is None: + # Be lenient to spaces after a delimiter, unless the + # delimiter is a space itself. + skipinitialspace = delimiter != ' ' + return reader(lines, delimiter=delimiter, + quotechar=quotechar or '"', + quoting=QUOTE_MINIMAL if quotechar else QUOTE_NONE, + escapechar=escapechar or None, + doublequote=doublequote, + skipinitialspace=skipinitialspace, + strict=True) + + def _try_dialect(self, lines, cut, delimiter, quotechar, escapechar): + """ + Parse the sample, pre-split into *lines*, and return the list + of the number of fields in every parsed row, or None if not a + single row was parsed. + + If the sample cannot be parsed to the end (for example it is + cut off in the middle of a quoted field, or the combination + does not fit the sample), the rows parsed so far are counted. + The last row is not counted if *cut* is true: the sample can + be cut off in the middle of it. + """ + rows = [] + try: + rows.extend(map(len, self._make_reader(lines, delimiter, + quotechar, escapechar))) + except Error: + # The row which failed to parse is not counted. + pass else: - doublequote = False + if cut and len(rows) > 1: + rows.pop() + if 0 in rows: + # Blank lines produce empty rows. + rows = [nfields for nfields in rows if nfields] + return rows or None + + def _eliminate_worse(self, parsed, judge_hopeless): + """ + Return the combinations from *parsed* (a list of (combination, + rows) pairs) without those which are clearly worse than the + leader. Combinations with too few parsed rows (e.g. if the + parsed part ends in the middle of a large quoted field) are + not judged yet. + + If *judge_hopeless* is false, keep the combinations whose + delimiter does not delimit anything. Unlike the comparison + with the leader, which self-normalizes when the parsed part is + not representative, this verdict is absolute and irreversible, + so it is not trusted to the first part, which covers the least + representative beginning of the sample (titles, headers, + preamble). + """ + # Judging a combination by fewer rows is too noisy. + min_rows = 16 + hopeless = set() + scores = {} + for combo, rows in parsed: + if rows is not None and len(rows) >= min_rows: + nfields, share = self._modal_share(rows) + if nfields < 2: + if judge_hopeless: + hopeless.add(combo) + else: + scores[combo] = share + threshold = max(scores.values(), default=0.0) - 0.1 + return [combo for combo, _ in parsed + if combo not in hopeless + and scores.get(combo, threshold) >= threshold] - return (quotechar, doublequote, delim, skipinitialspace) + def _modal_share(self, rows): + """ + The most common number of fields in a row and its share of all + rows. Prefer the smaller number of fields in the case of a + tie: a candidate delimiter which delimits only half of the rows + is not convincing. + """ + from collections import Counter + counts = Counter(rows) + nfields = max(counts, key=lambda n: (counts[n], -n)) + return nfields, counts[nfields] / len(rows) - def _guess_delimiter(self, data, delimiters): + def _detect_doublequote(self, lines, delimiter, quotechar, escapechar): """ - The delimiter /should/ occur the same number of times on - each row. However, due to malformed data, it may not. We don't want - an all or nothing approach, so we allow for small variations in this - number. - 1) build a table of the frequency of each character on every line. - 2) build a table of frequencies of this frequency (meta-frequency?), - e.g. 'x occurred 5 times in 10 rows, 6 times in 1000 rows, - 7 times in 2 rows' - 3) use the mode of the meta-frequency to determine the /expected/ - frequency for that character - 4) find out how often the character actually meets that goal - 5) the character that best meets its goal is the delimiter - For performance reasons, the data is evaluated in chunks, so it can - try and evaluate the smallest portion of the data possible, evaluating - additional chunks as necessary. + True if a doubled quote character represents a single quote + character in the sample: interpreting it so changes the result + of parsing. """ - from collections import Counter, defaultdict - - data = list(filter(None, data.split('\n'))) - - # build frequency tables - chunkLength = min(10, len(data)) - iteration = 0 - num_lines = 0 - # {char -> {count_per_line -> num_lines_with_that_count}} - char_frequency = defaultdict(Counter) - modes = {} - delims = {} - start, end = 0, chunkLength - while start < len(data): - iteration += 1 - for line in data[start:end]: - num_lines += 1 - for char, count in Counter(line).items(): - if char.isascii(): - char_frequency[char][count] += 1 - - for char, counts in char_frequency.items(): - items = list(counts.items()) - missed_lines = num_lines - sum(counts.values()) - if missed_lines: - # Store the number of lines 'char' was missing from. - items.append((0, missed_lines)) - if len(items) == 1 and items[0][0] == 0: - continue - # get the mode of the frequencies - if len(items) > 1: - modes[char] = max(items, key=lambda x: x[1]) - # adjust the mode - subtract the sum of all - # other frequencies - items.remove(modes[char]) - modes[char] = (modes[char][0], modes[char][1] - - sum(item[1] for item in items)) - else: - modes[char] = items[0] - - # build a list of possible delimiters - modeList = modes.items() - total = float(min(chunkLength * iteration, len(data))) - # (rows of consistent data) / (number of rows) = 100% - consistency = 1.0 - # minimum consistency threshold - threshold = 0.9 - while len(delims) == 0 and consistency >= threshold: - for k, v in modeList: - if v[0] > 0 and v[1] > 0: - if ((v[1]/total) >= consistency and - (delimiters is None or k in delimiters)): - delims[k] = v - consistency -= 0.01 - - if len(delims) == 1: - delim = list(delims.keys())[0] - skipinitialspace = (data[0].count(delim) == - data[0].count("%c " % delim)) - return (delim, skipinitialspace) - - # analyze another chunkLength lines - start = end - end += chunkLength - - if not delims: - return ('', 0) - - # if there's more than one, fall back to a 'preferred' list - if len(delims) > 1: - for d in self.preferred: - if d in delims.keys(): - skipinitialspace = (data[0].count(d) == - data[0].count("%c " % d)) - return (d, skipinitialspace) - - # nothing else indicates a preference, pick the character that - # dominates(?) - items = [(v,k) for (k,v) in delims.items()] - items.sort() - delim = items[-1][1] - - skipinitialspace = (data[0].count(delim) == - data[0].count("%c " % delim)) - return (delim, skipinitialspace) - + if not quotechar or not any(quotechar * 2 in line + for line in lines): + return False + readers = [self._make_reader( + lines, delimiter, quotechar, escapechar, + doublequote=doublequote) + for doublequote in (False, True)] + while True: + rows = [] + for rdr in readers: + try: + rows.append(next(rdr)) + except (StopIteration, Error): + # Ending cleanly and failing are equivalent here: + # after equal rows both readers are at the same + # position, so they cannot end for different + # reasons. + rows.append(None) + if rows[0] != rows[1]: + return True + if rows == [None, None]: + return False + + def _detect_skipinitialspace(self, lines, delimiter, quotechar, + escapechar, doublequote): + """ + True only if every field following a delimiter starts with + a space. + """ + skipinitialspace = False + try: + for row in self._make_reader(lines, delimiter, quotechar, + escapechar, + doublequote=doublequote, + skipinitialspace=False): + for field in row[1:]: + if not field.startswith(' '): + return False + skipinitialspace = True + except Error: + pass + return skipinitialspace def has_header(self, sample): # Creates a dictionary of types of data in each column. If any diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 7327c1bd5f5053..0e95b127320674 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1443,13 +1443,16 @@ def test_has_header_checks_20_rows(self): def test_guess_quote_and_delimiter(self): sniffer = csv.Sniffer() - for header in (";'123;4';", "'123;4';", ";'123;4'", "'123;4'"): + for header in (";'123;4';", "'123;4';", ";'123;4'"): with self.subTest(header): dialect = sniffer.sniff(header, ",;") self.assertEqual(dialect.delimiter, ';') self.assertEqual(dialect.quotechar, "'") self.assertIs(dialect.doublequote, False) self.assertIs(dialect.skipinitialspace, False) + # A single quoted field is a single column without a delimiter. + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, "'123;4'", ",;") def test_sniff(self): sniffer = csv.Sniffer() @@ -1495,6 +1498,196 @@ def test_delimiters(self): self.assertRaisesRegex(csv.Error, "Could not determine delimiter", sniffer.sniff, self.sample16) + def test_sniff_escapechar(self): + # gh-83273: escaped delimiters make the delimiter frequencies + # inconsistent, but the escape character can be detected by trial + # parsing. + sniffer = csv.Sniffer() + sample = 'ab,cd\\,ef\ngh\\,ij,kl\nmn,op\n' + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.escapechar, '\\') + self.assertIs(dialect.doublequote, False) + self.assertEqual(list(csv.reader(StringIO(sample), dialect)), + [['ab', 'cd,ef'], ['gh,ij', 'kl'], ['mn', 'op']]) + # No escape character in the sample -- none is detected. + dialect = sniffer.sniff('ab,cd\ngh,ij\n') + self.assertEqual(dialect.delimiter, ',') + self.assertIsNone(dialect.escapechar) + + def test_sniff_quoted_rows_among_unquoted(self): + # Rows which happen to consist of a single quoted or unquoted + # field must not be mistaken for a single column of quoted fields. + sniffer = csv.Sniffer() + sample = '"header line"\na|b\nc|d\ne|f\n' + self.assertEqual(sniffer.sniff(sample).delimiter, '|') + # Even if an unterminated quote breaks the quoted parse. + sample = '"header"\na|"b c\nd|e\nf|g\n' + self.assertEqual(sniffer.sniff(sample).delimiter, '|') + + def test_sniff_single_column_quoted(self): + # gh-98820: a sample consisting of a single column of quoted fields + # has no delimiter to detect, even if the quoted content contains + # characters which could pass for one. It also used to take + # quadratic time before failing. + sniffer = csv.Sniffer() + sample = '\n'.join('"%02d-%02d-%02d"' % (i, i, i) for i in range(50)) + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, sample) + # Even if none of the requested delimiters occurs in the sample. + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, sample, ',:|\t') + # A mix of quoted and unquoted single-field rows. + sample = '"abc"\ndef\n"ghi"\njkl\n' * 10 + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, sample) + # Even if the quoted fields contain delimiter characters. + sample = '"a,b"\ncd\n' * 20 + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, sample) + # Even if the content of the quoted fields parses consistently + # with the other quote character. + sample = '''"a-'b'-c"\n"d-'e'-f"\n''' * 10 + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, sample) + + def test_sniff_non_ascii_delimiter(self): + # gh-111820: an explicitly requested delimiter can be non-ASCII. + sniffer = csv.Sniffer() + sample = 'aaa\xacbbb\xacccc\nddd\xaceee\xacfff\nggg\xachhh\xaciii\n' + dialect = sniffer.sniff(sample, delimiters='\xac') + self.assertEqual(dialect.delimiter, '\xac') + + def test_sniff_preamble(self): + # A preamble (title or comment lines) before the data must not + # prevent detection, even if it is larger than the part of the + # sample which is parsed first. It is enough for the data rows + # to slightly outnumber the preamble lines. + sniffer = csv.Sniffer() + for n in 3, 24, 80, 320: + preamble = ''.join(f'Comment line {i:05}\n' for i in range(n)) + for ndata in n + 2, 2 * n + 5: + with self.subTest(preamble_lines=n, data_lines=ndata): + sample = preamble + 'aaa,bbb,ccc\n' * ndata + self.assertEqual(sniffer.sniff(sample).delimiter, ',') + + def test_sniff_line_boundary_characters(self): + # Only '\r', '\n' and '\r\n' are row separators; characters like + # '\x1c' or '\x85', which str.splitlines() treats as line + # boundaries, are ordinary data for the reader -- '\x1c' can + # even be the delimiter. + sniffer = csv.Sniffer() + sample = 'aa\x1cbb\ncc\x1cdd\nee\x1cff\n' + self.assertEqual(sniffer.sniff(sample).delimiter, '\x1c') + sample = 'a\x85b,c\nd,e\nf,g\n' + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(next(csv.reader(StringIO(sample), dialect)), + ['a\x85b', 'c']) + + def test_sniff_delimiter_in_quoted_field(self): + # gh-97611: a delimiter inside a quoted field should not win over + # the delimiter which actually separates the fields. + sniffer = csv.Sniffer() + sample = ( + 'Surname;First Name;Year of birth\n' + '"\tDoe;Jane"\t;1971\n' + ) + dialect = sniffer.sniff(sample, delimiters=',;\t') + self.assertEqual(dialect.delimiter, ';') + sample = ( + 'Surname;First Name;Year of birth\n' + '"\t";"\t";1971\n' + '"Le Trec";"Mary Ann";1486\n' + ) + dialect = sniffer.sniff(sample, delimiters=',;\t') + self.assertEqual(dialect.delimiter, ';') + self.assertEqual(dialect.quotechar, '"') + + def test_sniff_embedded_lists(self): + # gh-119123: commas in bracketed lists adjacent to quotes should + # not be mistaken for the delimiter. + sniffer = csv.Sniffer() + sample = ( + "id;is_sort;cost;group;merge\n" + "1;True;62.25;['345'];UNKNOWN\n" + "2;True;54.00;['235'];UNKNOWN\n" + "3;True;237.00;['567', '568'];UNKNOWN\n" + "4;True;46.50;['112', '112'];UNKNOWN\n" + ) + dialect = sniffer.sniff(sample, delimiters=',;') + self.assertEqual(dialect.delimiter, ';') + + def test_sniff_space_adjacent_to_quotes(self): + # gh-88843: a space adjacent to stray quotes should not be + # detected as the delimiter. + sniffer = csv.Sniffer() + sample = "a|b\nc| 'd\ne|' f" + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, '|') + + def test_sniff_crlf_lineterminator(self): + # gh-103925: a quote at the end of a \r\n-terminated line. + sniffer = csv.Sniffer() + sample = ( + 'Timestamp,URL,Title\r\n' + '2020-10-01 17:17:37+08:00,' + 'https://www.mozilla.org/en-US/firefox/welcome/2/,' + '"Pocket - Save news, videos, stories and more"\r\n' + ) + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.quotechar, '"') + + def test_sniff_excel_tab_with_quotes(self): + # gh-62029: tab-delimited data with a quoted field containing + # spaces and doubled quotes. + sniffer = csv.Sniffer() + sample = ('foo\tbar\t"baz ""quoted"" here"\tspam eggs\n' + 'ham\teggs\tx y\tz w\n') + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, '\t') + self.assertEqual(dialect.quotechar, '"') + self.assertIs(dialect.doublequote, True) + + def test_sniff_truncated_sample(self): + # A sample cut off in the middle of a quoted field should not + # spoil the detection. + sniffer = csv.Sniffer() + sample = '"a,a";"b"\n"c";"d"\n"e";"f,f\n' + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ';') + self.assertEqual(dialect.quotechar, '"') + # Cut off in the middle of the last row. + sample = 'a,b,c\nd,e,f\ng,h,i\nj,k' + self.assertEqual(sniffer.sniff(sample).delimiter, ',') + # Cut off in the middle of an escaped sequence. + sample = 'ab,cd\\,ef\ngh\\,ij,kl\nmn,op\nqr,st\\' + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.escapechar, '\\') + # Cut off at a line end in the middle of a quoted field. + sample = '"a","b"\n"c","d"\n"e","multi\nline' + self.assertEqual(sniffer.sniff(sample).delimiter, ',') + # An unclosed quote consumes everything to the end of the sample, + # but the rows before it still count. + sample = 'a|b\nc|d\ne|f\ng|"h\ni|j\nk|l' + self.assertEqual(sniffer.sniff(sample).delimiter, '|') + + def test_sniff_skipinitialspace_quoted(self): + sniffer = csv.Sniffer() + sample = "'a': 'b': 'c'\n'd': 'e': 'f'\n" + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ':') + self.assertEqual(dialect.quotechar, "'") + self.assertIs(dialect.skipinitialspace, True) + + def test_sniff_regex_backtracking(self): + # gh-109638: this artificial sample used to take minutes. + sniffer = csv.Sniffer() + sample = '"",' * 100 + '"' * 100 + '0' + '"' * 100 + '0' + self.assertEqual(sniffer.sniff(sample).delimiter, ',') + def test_doublequote(self): sniffer = csv.Sniffer() dialect = sniffer.sniff(self.header1) @@ -1507,6 +1700,15 @@ def test_doublequote(self): self.assertFalse(dialect.doublequote) dialect = sniffer.sniff(self.sample9) self.assertTrue(dialect.doublequote) + # A doubled quote character in an unquoted field or an empty + # quoted field is not evidence of doubling. + dialect = sniffer.sniff('"x",a""b\n"y",c\n') + self.assertIs(dialect.doublequote, False) + dialect = sniffer.sniff('a,"",c\nd,"",f\n') + self.assertIs(dialect.doublequote, False) + # A doubled quote character inside a quoted field is. + dialect = sniffer.sniff('"a""b",c\n"d",e\n') + self.assertIs(dialect.doublequote, True) def test_guess_delimiter_crlf_not_chosen(self): # Ensure that we pick the real delimiter ("|") over "\r" in a tie. diff --git a/Misc/NEWS.d/next/Library/2026-07-14-12-00-00.gh-issue-83273.SnifQ7.rst b/Misc/NEWS.d/next/Library/2026-07-14-12-00-00.gh-issue-83273.SnifQ7.rst new file mode 100644 index 00000000000000..789503913048f9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-14-12-00-00.gh-issue-83273.SnifQ7.rst @@ -0,0 +1,10 @@ +:meth:`csv.Sniffer.sniff` now deduces the dialect by trial parsing with the +actual CSV parser instead of heuristics based on matching isolated fragments +and on character frequencies. It can now +detect the *escapechar* parameter, accepts non-ASCII delimiters in the +*delimiters* argument, no longer misdetects the delimiter when the sample +contains delimiter characters inside quoted fields, handles samples +truncated at an arbitrary point, and no longer takes quadratic time on +quoted samples. A sample consisting of a single column of quoted fields +now raises :exc:`csv.Error` instead of guessing a delimiter from the +content of the fields.