diff --git a/csv_diff/__init__.py b/csv_diff/__init__.py index 59a2eaf..51b4ff2 100644 --- a/csv_diff/__init__.py +++ b/csv_diff/__init__.py @@ -1,10 +1,19 @@ import csv +import sys from dictdiffer import diff import json import hashlib def load_csv(fp, key=None, dialect=None): + # The C parser's per-field size cap defaults to 131072 bytes, which is + # far too small for CSVs that contain long strings (e.g. nucleotide + # sequences, large JSON blobs, embedded log lines). When the limit is + # exceeded the parser raises ``_csv.Error: field larger than field + # limit (131072)`` instead of returning the row. Bump the limit to + # the platform's ``PY_SSIZE_T_MAX`` so the parser can handle long + # fields. See issue #41. + csv.field_size_limit(sys.maxsize) if dialect is None and fp.seekable(): # Peek at first 1MB to sniff the delimiter and other dialect details peek = fp.read(1024**2) diff --git a/tests/test_csv_diff.py b/tests/test_csv_diff.py index 0e3670f..badfdbd 100644 --- a/tests/test_csv_diff.py +++ b/tests/test_csv_diff.py @@ -115,3 +115,25 @@ def test_tsv(): "columns_added": [], "columns_removed": [], } == diff + + +def test_load_csv_handles_fields_larger_than_default_limit(): + # Regression test for https://github.com/simonw/csv-diff/issues/41 + # The csv module's default per-field size cap is 131072 bytes; rows with + # longer fields (e.g. nucleotide sequences, large JSON blobs) used to + # raise ``_csv.Error: field larger than field limit (131072)`` from + # ``load_csv``. ``load_csv`` now bumps the cap to ``sys.maxsize`` so + # these rows are returned in full. + long_value = "A" * (200_000) + csv_text = f"id,sequence\n1,{long_value}\n" + rows = load_csv(io.StringIO(csv_text), key="id") + assert rows == {"1": {"id": "1", "sequence": long_value}} + + +def test_load_csv_handles_tsv_with_long_fields(): + # Same fix as above, exercising the TSV path (which uses the same + # underlying csv reader and is also subject to the size cap). + long_value = "T" * (200_000) + tsv_text = f"id\tsequence\n1\t{long_value}\n" + rows = load_csv(io.StringIO(tsv_text), key="id") + assert rows == {"1": {"id": "1", "sequence": long_value}}