fix(serialization): write specs as strict JSON without NaN tokens

docs/core_concepts/reproducibility.md

@@ -29,6 +29,14 @@ directly; they are fit outside `Preprocessor` since they operate on the whole fe
 reconstructs it. The spec records the schema and library versions, the resolved parameters,
 and the per-representation fitted state (parameters, knots, centers, columns, scaling).
 
+The spec is strict JSON, so any standard parser (JavaScript, Go, Rust, JSON schema validators)
+reads it. Non-finite floats, such as the imputers' `missing_values=np.nan`, are stored as a
+tagged value, `{"__float__": "nan"}` (or `"inf"`, `"-inf"`), instead of a bare `NaN` token.
+Specs written by PreTab 1.0.0, which contain bare `NaN` tokens, still load.
+The reverse is not supported: older readers cannot decode the new float tags.
+The tagged encoding also changes fingerprints for fitted state containing non-finite
+values, including the default imputers; this change does not alter transformed values.
+
 ```python
 spec = pre.to_spec()                 # returns a dict
 pre.to_spec("representation.json")   # or writes JSON to a path

pretab/compose/serialize.py

@@ -12,10 +12,16 @@
 constructor params, per-representation summary, output column order) plus a
 ``state`` payload that JSON-encodes the fitted object graph (numpy arrays, knot /
 center / bin locations, scalers, nested estimators) for exact reconstruction.
+
+The document is strict (RFC 8259) JSON: non-finite floats, such as an imputer's
+``missing_values=np.nan``, are tagged as ``{"__float__": "nan"}`` (``"inf"``,
+``"-inf"``), so parsers outside Python read it. Bare ``NaN`` tokens written by
+earlier versions are still decoded.
 """
 
 import dataclasses
 import importlib
+import math
 from collections import UserList
 from typing import Any, cast
 
@@ -89,14 +95,23 @@ def _resolve(dotted: str):
 
 
 # --- encoding ------------------------------------------------------------
+def _encode_float(value):
+    """Return ``value``, tagging NaN / +-inf, which strict JSON cannot represent."""
+    if math.isfinite(value):
+        return value
+    return {"__float__": repr(float(value))}
+
+
 def _encode(obj):
     if isinstance(obj, np.bool_):
         return bool(obj)
     if isinstance(obj, np.integer):
         return int(obj)
     if isinstance(obj, np.floating):
-        return float(obj)
-    if obj is None or isinstance(obj, (bool, int, float, str)):
+        return _encode_float(float(obj))
+    if isinstance(obj, float):
+        return _encode_float(obj)
+    if obj is None or isinstance(obj, (bool, int, str)):
         return obj
     if obj is UNSET:
         return {"__unset__": True}
@@ -106,6 +121,8 @@ def _encode(obj):
             # tolist() keeps the elements of an object array as they are (e.g.
             # numpy scalars such as np.bool_), so encode each one.
             data = _map_elements(data, obj.ndim, _encode)
+        elif obj.dtype.kind == "f" and not np.isfinite(obj).all():
+            data = _map_elements(data, obj.ndim, _encode_float)
         return {"__ndarray__": {"dtype": obj.dtype.str, "shape": list(obj.shape), "data": data}}
     if isinstance(obj, np.dtype):
         return {"__npdtype__": obj.str}
@@ -152,7 +169,9 @@ def _json_safe(obj):
     Keeps JSON-native values verbatim and falls back to the tagged :func:`_encode`
     form only for exotic values. The result is informational and never decoded.
     """
-    if obj is None or isinstance(obj, (bool, int, float, str)):
+    if isinstance(obj, float):
+        return _encode_float(obj)
+    if obj is None or isinstance(obj, (bool, int, str)):
         return obj
     if isinstance(obj, dict) and all(isinstance(k, str) for k in obj):
         return {k: _json_safe(v) for k, v in obj.items()}
@@ -174,7 +193,10 @@ def _decode_ndarray(payload: dict) -> np.ndarray:
         for index, element in enumerate(elements):
             arr[index] = element
         return arr.reshape(shape)
-    arr = np.array(payload["data"], dtype=dtype)
+    data = payload["data"]
+    if dtype.kind == "f":
+        data = _map_elements(data, len(payload["shape"]), _decode)
+    arr = np.array(data, dtype=dtype)
     return arr.reshape(payload["shape"])
 
 
@@ -188,6 +210,8 @@ def _decode(obj):
             return _decode_ndarray(obj["__ndarray__"])
         if "__unset__" in obj:
             return UNSET
+        if "__float__" in obj:
+            return float(obj["__float__"])
         if "__npdtype__" in obj:
             return np.dtype(obj["__npdtype__"])
         if "__type__" in obj:

pretab/preprocessor.py

@@ -112,9 +112,9 @@ def _preset_numerical_method(task) -> str:
 
 
 def _spec_json(spec: dict, **kwargs) -> str:
-    """Serialize a spec to JSON text, raising a typed error for unsupported state."""
+    """Serialize a spec to strict JSON, raising a typed error for unsupported state."""
     try:
-        return json.dumps(spec, **kwargs)
+        return json.dumps(spec, allow_nan=False, **kwargs)
     except (TypeError, ValueError) as exc:
         raise PretabSerializationError(f"The fitted state cannot be written as JSON: {exc}") from exc
 
@@ -1355,7 +1355,10 @@ def to_spec(self, path=None) -> dict:
         Parameters
         ----------
         path : str, os.PathLike, or None, default=None
-            When given, the spec is also written to this path as UTF-8 JSON.
+            When given, the spec is also written to this path as strict UTF-8 JSON:
+            non-finite floats are stored as ``{"__float__": "nan"}`` (or ``"inf"`` /
+            ``"-inf"``), so the file contains no ``NaN`` / ``Infinity`` tokens and
+            any standard JSON parser reads it.
 
         Returns
         -------
@@ -1387,7 +1390,9 @@ def from_spec(cls, source) -> "Preprocessor":
         Parameters
         ----------
         source : str, os.PathLike, or dict
-            A path to a JSON spec file, or the spec dictionary itself.
+            A path to a JSON spec file, or the spec dictionary itself. Specs written
+            by PreTab 1.0.0, which store non-finite floats as bare ``NaN`` tokens,
+            still load.
 
         Returns
         -------

tests/integration/test_serialization.py

@@ -7,6 +7,8 @@
 """
 
 import json
+import math
+from typing import cast
 
 import numpy as np
 import pandas as pd
@@ -328,6 +330,124 @@ def test_from_spec_rejects_bad_source_type():
         Preprocessor.from_spec(12345)
 
 
+# --- strict JSON: no NaN / Infinity tokens (issue #80) -----------------------------
+
+
+def _strict_loads(text):
+    def reject(token):
+        raise ValueError(f"non-standard JSON token {token}")
+
+    return json.loads(text, parse_constant=reject)
+
+
+@pytest.fixture
+def frame_with_missing():
+    rng = np.random.default_rng(0)
+    a = rng.normal(size=80)
+    a[:6] = np.nan
+    X = pd.DataFrame({"a": a, "b": rng.exponential(size=80), "c": rng.choice(["x", "y"], 80)})
+    return X, rng.normal(size=80)
+
+
+def test_default_spec_file_is_strict_json(frame_with_missing, tmp_path):
+    """The imputers' NaN missing-value marker was written as a bare NaN token,
+    which strict JSON parsers (JavaScript, Go, Rust, ...) reject."""
+    X, y = frame_with_missing
+    pre = Preprocessor(random_state=0).fit(X, y)
+    path = tmp_path / "spec.json"
+    fingerprint = pre.fingerprint_
+    pre.to_spec(path)
+    text = path.read_text(encoding="utf-8")
+    pre.to_spec(path)
+    assert path.read_text(encoding="utf-8") == text
+    assert pre.fingerprint_ == fingerprint
+    assert "NaN" not in text and "Infinity" not in text
+    loaded = Preprocessor.from_spec(_strict_loads(text))
+    np.testing.assert_array_equal(np.asarray(loaded.transform(X)), np.asarray(pre.transform(X)))
+    assert loaded.fingerprint_ == pre.fingerprint_
+
+
+def test_nan_category_in_an_object_array_is_strict_json():
+    """A NaN category of an un-imputed one-hot encoder sits in an object array."""
+    rng = np.random.default_rng(0)
+    city = rng.choice(["x", "y", "z"], 80).astype(object)
+    city[:5] = np.nan
+    X = pd.DataFrame({"a": rng.normal(size=80), "city": city})
+    pre = Preprocessor(categorical_method="one-hot", categorical_imputation=None, random_state=0).fit(X, X["a"])
+    text = json.dumps(pre.to_spec(), allow_nan=False)
+    assert '{"__float__": "nan"}' in text
+    loaded = Preprocessor.from_spec(_strict_loads(text))
+    np.testing.assert_array_equal(np.asarray(loaded.transform(X)), np.asarray(pre.transform(X)))
+    assert loaded.fingerprint_ == pre.fingerprint_
+
+
+def test_non_finite_floats_round_trip():
+    from pretab.compose.serialize import _decode, _encode
+
+    array = np.array([[1.0, np.nan], [np.inf, -np.inf]])
+    decoded = _decode(_strict_loads(json.dumps(_encode(array), allow_nan=False)))
+    assert isinstance(decoded, np.ndarray)
+    np.testing.assert_array_equal(decoded, array)
+    assert decoded.dtype == array.dtype
+    nan, neg_inf = (
+        float(cast(float, _decode(_strict_loads(json.dumps(_encode(value), allow_nan=False)))))
+        for value in (np.nan, -np.inf)
+    )
+    assert math.isnan(nan) and neg_inf == -math.inf
+
+
+@pytest.mark.parametrize("dtype", [np.float32, np.float64])
+@pytest.mark.parametrize("shape", [(), (4,), (2, 2), (0, 2)])
+def test_strict_json_float_array_shapes_and_dtypes_round_trip(dtype, shape):
+    from pretab.compose.serialize import _decode, _encode
+
+    array = np.resize(np.array([np.nan, np.inf, -np.inf, 1.0], dtype=dtype), shape)
+    decoded = _decode(_strict_loads(json.dumps(_encode(array), allow_nan=False)))
+
+    assert isinstance(decoded, np.ndarray)
+    assert decoded.shape == shape
+    assert decoded.dtype == array.dtype
+    np.testing.assert_array_equal(decoded, array)
+
+
+@pytest.mark.parametrize("shape", [(), (6,), (2, 3), (0, 2)])
+def test_strict_json_object_arrays_preserve_nested_elements(shape):
+    from pretab.compose.serialize import _decode, _encode
+
+    elements = [
+        [np.bool_(True), np.float32(np.nan)],
+        (np.int64(7), np.float64(np.inf)),
+        np.array([np.nan, -np.inf], dtype=np.float32),
+        {"missing": np.float32(np.nan)},
+        np.bool_(False),
+        np.int64(3),
+    ]
+    array = np.empty(shape, dtype=object)
+    for index in range(array.size):
+        array.flat[index] = elements[index]
+    decoded = _decode(_strict_loads(json.dumps(_encode(array), allow_nan=False)))
+
+    assert isinstance(decoded, np.ndarray)
+    assert decoded.shape == shape
+    assert decoded.dtype == object
+    for actual, expected in zip(decoded.flat, array.flat, strict=True):
+        assert type(actual) is type(expected) or isinstance(expected, np.generic)
+        np.testing.assert_equal(actual, expected)
+        if isinstance(expected, np.ndarray):
+            assert actual.dtype == expected.dtype
+
+
+def test_specs_with_bare_nan_tokens_still_load(frame_with_missing):
+    X, y = frame_with_missing
+    pre = Preprocessor(random_state=0).fit(X, y)
+    spec = json.loads(json.dumps(pre.to_spec()))
+    # Rewrite the tagged floats the way older versions wrote them: as bare NaN.
+    legacy_text = json.dumps(spec).replace('{"__float__": "nan"}', "NaN")
+    assert "NaN" in legacy_text
+    loaded = Preprocessor.from_spec(json.loads(legacy_text))
+    np.testing.assert_array_equal(np.asarray(loaded.transform(X)), np.asarray(pre.transform(X)))
+
+
 # --- object arrays and builtin dtypes (issue #66) ---------------------------------
 
 
@@ -362,7 +482,8 @@ def test_builtin_and_numpy_dtypes_round_trip(dtype):
     assert loaded.fingerprint_ == pre.fingerprint_
 
 
-def test_failed_save_leaves_an_existing_spec_untouched(tmp_path, monkeypatch):
+@pytest.mark.parametrize("unsupported", [object(), np.nan, np.inf, -np.inf], ids=["object", "nan", "inf", "neg_inf"])
+def test_failed_save_leaves_an_existing_spec_untouched(tmp_path, monkeypatch, unsupported):
     import pretab.preprocessor as preprocessor_module
 
     frame = pd.DataFrame({"age": np.linspace(20.0, 60.0, 30)})
@@ -375,11 +496,13 @@ def test_failed_save_leaves_an_existing_spec_untouched(tmp_path, monkeypatch):
     # already-opened file used to leave a truncated, invalid spec behind.
     spec = pre.to_spec()
     monkeypatch.setattr(
-        preprocessor_module, "preprocessor_to_spec", lambda _: {**spec, "state": {"unsupported_": object()}}
+        preprocessor_module, "preprocessor_to_spec", lambda _: {**spec, "state": {"unsupported_": unsupported}}
     )
     with pytest.raises(PretabSerializationError, match="cannot be written as JSON"):
         pre.to_spec(path)
     assert path.read_text(encoding="utf-8") == saved
+    with pytest.raises(PretabSerializationError, match="cannot be written as JSON"):
+        _ = pre.fingerprint_
 
 
 def test_types_a_spec_cannot_load_are_rejected_at_save_time():