Merge pull request #120 from FreshCode-Org/fix/int64-boundary-finalize-numeric-jwd

fix: exact int64 boundary handling in numeric finalization (#34)

Author: JohnnyWilson16

CHANGELOG.md

@@ -15,6 +15,11 @@ adheres to [Semantic Versioning](https://semver.org/).
   count, and a dtype conversion alone no longer marks untouched values as
   changed. The elementwise fallback also no longer uses a Python-3.10-only
   `zip(strict=...)` argument, which crashed on Python 3.9 when reached (#30).
+- Integer finalization now checks the exact int64 range in integer space
+  instead of a float magnitude threshold: `-2**63` and `2**63 - 1024` (the
+  largest float64 below `2**63`) convert to int64/Int64 exactly instead of
+  being demoted to float64, and values at or above `2**63` can never be
+  admitted by float rounding (#34).
 
 ### Added
 - **AI Copilot (experimental)** — `freshdata.experimental.ai_copilot.analyze_dataset`:

src/freshdata/steps/dtypes.py

@@ -82,7 +82,25 @@ def _has_leading_zero_ids(sample: pd.Series) -> bool:
 )
 
 # Largest float that safely round-trips through int64.
-_INT64_SAFE = float(2**63 - 1024)
+# int64's exact range. Bounds are compared in Python-int space (see
+# _fits_int64): float64 cannot represent 2**63 - 1 — it rounds up to 2**63 —
+# so any float-space threshold either rejects legitimate boundary values or
+# admits one that overflows .astype("int64").
+_INT64_MIN = -(2**63)
+_INT64_MAX = 2**63 - 1
+
+
+def _fits_int64(nonnull: pd.Series) -> bool:
+    """True when every value converts to int64 without wrapping.
+
+    Callers guarantee the series is integral (``% 1 == 0``), so ``int()`` on
+    the min/max is an exact conversion; ``inf`` raises OverflowError and is
+    reported as not fitting.
+    """
+    try:
+        return int(nonnull.min()) >= _INT64_MIN and int(nonnull.max()) <= _INT64_MAX
+    except (OverflowError, ValueError):
+        return False
 
 
 def _finalize_numeric(parsed: pd.Series) -> pd.Series:
@@ -91,7 +109,7 @@ def _finalize_numeric(parsed: pd.Series) -> pd.Series:
     is_integral = (
         len(nonnull) > 0
         and bool((nonnull % 1 == 0).all())
-        and float(nonnull.abs().max()) < _INT64_SAFE
+        and _fits_int64(nonnull)
     )
     if is_integral:
         return parsed.astype("int64") if not parsed.isna().any() else parsed.astype("Int64")

tests/test_dtypes.py

@@ -5,6 +5,7 @@
 import pytest
 
 import freshdata as fd
+from freshdata.steps.dtypes import _finalize_numeric
 
 
 def clean1(values, **options):
@@ -156,3 +157,43 @@ def test_date_objects_normalized_to_datetime64():
 def test_huge_integers_stay_float_not_overflow(huge):
     s = clean1(huge)
     assert s.dtype == "float64"
+
+
+def test_finalize_numeric_int64_boundaries_no_overflow_no_demotion():
+    """Regression for #34: exact int64 boundary handling.
+
+    float64 cannot represent 2**63 - 1; it rounds up to 2**63, so any
+    float-space threshold either rejects legitimate values or admits an
+    overflowing one.  The largest float64 below 2**63 is 2**63 - 1024 and
+    must convert exactly; float(2**63) must stay float64, never wrap.
+    """
+    top = float(2**63 - 1024)
+    out = _finalize_numeric(pd.Series([top, 1.0]))
+    assert str(out.dtype) == "int64"
+    assert int(out.iloc[0]) == 2**63 - 1024
+
+    out = _finalize_numeric(pd.Series([float(2**63), 1.0]))
+    assert str(out.dtype) == "float64"
+
+    out = _finalize_numeric(pd.Series([top, None]))
+    assert str(out.dtype) == "Int64"
+    assert int(out.iloc[0]) == 2**63 - 1024
+
+
+def test_finalize_numeric_int64_min_not_rejected_by_abs_asymmetry():
+    """int64's range is asymmetric: -2**63 is representable, +2**63 is not.
+    A magnitude-only guard rejected the legitimate minimum."""
+    bottom = float(-(2**63))
+    out = _finalize_numeric(pd.Series([bottom, 0.0]))
+    assert str(out.dtype) == "int64"
+    assert int(out.iloc[0]) == -(2**63)
+
+    out = _finalize_numeric(pd.Series([float(-(2**64)), 0.0]))
+    assert str(out.dtype) == "float64"
+
+
+def test_huge_integer_strings_still_stay_float_end_to_end():
+    """Pipeline-level guard: values beyond int64 parse to float64, exactly
+    as before the boundary fix."""
+    s = clean1(["18446744073709551616", "1"])  # 2**64
+    assert s.dtype == "float64"