From c5b5c217b8530050dde0bca89a4e7ee15dd2530c Mon Sep 17 00:00:00 2001 From: ChrisW09 <50968720+ChrisW09@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:20:55 +0200 Subject: [PATCH] fix(serialization): write specs as strict JSON without NaN tokens The imputers' missing_values=np.nan (and any other non-finite float in fitted state) was written as a bare NaN token, so every default spec file was rejected by strict JSON parsers outside Python (JavaScript, Go, Rust, JSON schema validators). The encoder passed floats and float arrays through unchanged, and the spec was dumped with the default allow_nan=True. Non-finite floats, as scalars, as float-array elements and as float elements of object arrays (e.g. a NaN category of an un-imputed one-hot encoder), are now encoded as a tagged value such as {"__float__": "nan"} and decoded back, and specs are dumped with allow_nan=False, both by to_spec(path) and for fingerprint_. Specs written by earlier versions, with bare NaN tokens, still load and reproduce the same output. fingerprint_ changes for every fitted preprocessor whose state holds a non-finite float (every one with an imputer). The to_spec / from_spec docstrings, the serialize module docstring and the reproducibility guide describe the strict format. Fixes #80 --- docs/core_concepts/reproducibility.md | 5 ++ pretab/compose/serialize.py | 58 +++++++++++++++++-- pretab/preprocessor.py | 17 ++++-- tests/integration/test_serialization.py | 75 +++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 9 deletions(-) diff --git a/docs/core_concepts/reproducibility.md b/docs/core_concepts/reproducibility.md index ec6f1e3..31b555b 100644 --- a/docs/core_concepts/reproducibility.md +++ b/docs/core_concepts/reproducibility.md @@ -29,6 +29,11 @@ directly; they are fit outside `Preprocessor` since they operate on the whole fe reconstructs it. The spec records the schema and library versions, the resolved parameters, and the per-representation fitted state (parameters, knots, centers, columns, scaling). +The spec is strict JSON, so any standard parser (JavaScript, Go, Rust, JSON schema validators) +reads it. Non-finite floats, such as the imputers' `missing_values=np.nan`, are stored as a +tagged value, `{"__float__": "nan"}` (or `"inf"`, `"-inf"`), instead of a bare `NaN` token. +Specs written by PreTab 1.0.0, which contain bare `NaN` tokens, still load. + ```python spec = pre.to_spec() # returns a dict pre.to_spec("representation.json") # or writes JSON to a path diff --git a/pretab/compose/serialize.py b/pretab/compose/serialize.py index c4b213f..fd144a9 100644 --- a/pretab/compose/serialize.py +++ b/pretab/compose/serialize.py @@ -12,10 +12,16 @@ constructor params, per-representation summary, output column order) plus a ``state`` payload that JSON-encodes the fitted object graph (numpy arrays, knot / center / bin locations, scalers, nested estimators) for exact reconstruction. + +The document is strict (RFC 8259) JSON: non-finite floats, such as an imputer's +``missing_values=np.nan``, are tagged as ``{"__float__": "nan"}`` (``"inf"``, +``"-inf"``), so parsers outside Python read it. Bare ``NaN`` tokens written by +earlier versions are still decoded. """ import dataclasses import importlib +import math from collections import UserList from typing import Any, cast @@ -53,6 +59,13 @@ # --- helpers ------------------------------------------------------------- +def _map_elements(data, depth: int, func): + """Apply ``func`` to the elements of a ``depth``-level nested list (an array's ``tolist()``).""" + if depth == 0: + return func(data) + return [_map_elements(item, depth - 1, func) for item in data] + + def _qualname(obj) -> str: cls = type(obj) return f"{cls.__module__}:{cls.__qualname__}" @@ -77,19 +90,40 @@ def _resolve(dotted: str): # --- encoding ------------------------------------------------------------ +def _encode_float(value): + """Return ``value``, tagging NaN / +-inf, which strict JSON cannot represent.""" + if math.isfinite(value): + return value + return {"__float__": repr(float(value))} + + +def _encode_object_element(value): + """Tag a non-finite float element of an object array; keep any other element as it is.""" + return _encode_float(value) if isinstance(value, float) else value + + def _encode(obj): if isinstance(obj, np.bool_): return bool(obj) if isinstance(obj, np.integer): return int(obj) if isinstance(obj, np.floating): - return float(obj) - if obj is None or isinstance(obj, (bool, int, float, str)): + return _encode_float(float(obj)) + if isinstance(obj, float): + return _encode_float(obj) + if obj is None or isinstance(obj, (bool, int, str)): return obj if obj is UNSET: return {"__unset__": True} if isinstance(obj, np.ndarray): - return {"__ndarray__": {"dtype": obj.dtype.str, "shape": list(obj.shape), "data": obj.tolist()}} + data = obj.tolist() + if obj.dtype.kind == "f" and not np.isfinite(obj).all(): + data = _map_elements(data, obj.ndim, _encode_float) + elif obj.dtype.kind == "O": + # tolist() keeps the elements of an object array as they are, so tag + # the non-finite floats among them (e.g. a NaN category). + data = _map_elements(data, obj.ndim, _encode_object_element) + return {"__ndarray__": {"dtype": obj.dtype.str, "shape": list(obj.shape), "data": data}} if isinstance(obj, np.dtype): return {"__npdtype__": obj.str} if isinstance(obj, type): @@ -126,7 +160,9 @@ def _json_safe(obj): Keeps JSON-native values verbatim and falls back to the tagged :func:`_encode` form only for exotic values. The result is informational and never decoded. """ - if obj is None or isinstance(obj, (bool, int, float, str)): + if isinstance(obj, float): + return _encode_float(obj) + if obj is None or isinstance(obj, (bool, int, str)): return obj if isinstance(obj, dict) and all(isinstance(k, str) for k in obj): return {k: _json_safe(v) for k, v in obj.items()} @@ -136,9 +172,19 @@ def _json_safe(obj): # --- decoding ------------------------------------------------------------ +def _decode_float_element(item): + """Restore a non-finite float tagged by :func:`_encode_float`; keep any other element.""" + if isinstance(item, dict) and "__float__" in item: + return float(item["__float__"]) + return item + + def _decode_ndarray(payload: dict) -> np.ndarray: dtype = np.dtype(payload["dtype"]) - arr = np.array(payload["data"], dtype=dtype) + data = payload["data"] + if dtype.kind in "fO": + data = _map_elements(data, len(payload["shape"]), _decode_float_element) + arr = np.array(data, dtype=dtype) return arr.reshape(payload["shape"]) @@ -152,6 +198,8 @@ def _decode(obj): return _decode_ndarray(obj["__ndarray__"]) if "__unset__" in obj: return UNSET + if "__float__" in obj: + return float(obj["__float__"]) if "__npdtype__" in obj: return np.dtype(obj["__npdtype__"]) if "__type__" in obj: diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index a8193b8..f728d28 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -1009,7 +1009,10 @@ def to_spec(self, path=None) -> dict: Parameters ---------- path : str, os.PathLike, or None, default=None - When given, the spec is also written to this path as UTF-8 JSON. + When given, the spec is also written to this path as strict UTF-8 JSON: + non-finite floats are stored as ``{"__float__": "nan"}`` (or ``"inf"`` / + ``"-inf"``), so the file contains no ``NaN`` / ``Infinity`` tokens and + any standard JSON parser reads it. Returns ------- @@ -1020,7 +1023,9 @@ def to_spec(self, path=None) -> dict: spec = preprocessor_to_spec(self) if path is not None: with open(path, "w", encoding="utf-8") as handle: - json.dump(spec, handle, indent=2) + # Non-finite floats are tagged by the encoder; allow_nan=False keeps + # bare NaN / Infinity tokens (rejected by strict JSON parsers) out. + json.dump(spec, handle, indent=2, allow_nan=False) return spec @classmethod @@ -1030,7 +1035,9 @@ def from_spec(cls, source) -> "Preprocessor": Parameters ---------- source : str, os.PathLike, or dict - A path to a JSON spec file, or the spec dictionary itself. + A path to a JSON spec file, or the spec dictionary itself. Specs written + by PreTab 1.0.0, which store non-finite floats as bare ``NaN`` tokens, + still load. Returns ------- @@ -1080,7 +1087,9 @@ def fingerprint_(self) -> str: the same fitted state and configuration produce the same fingerprint. """ check_is_fitted(self) - canonical = json.dumps(self._canonical_spec(), sort_keys=True, separators=(",", ":"), ensure_ascii=True) + canonical = json.dumps( + self._canonical_spec(), sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False + ) return hashlib.sha256(canonical.encode("utf-8")).hexdigest() def reproducibility_report(self) -> dict: diff --git a/tests/integration/test_serialization.py b/tests/integration/test_serialization.py index 493000c..f0807fc 100644 --- a/tests/integration/test_serialization.py +++ b/tests/integration/test_serialization.py @@ -7,6 +7,8 @@ """ import json +import math +from typing import cast import numpy as np import pandas as pd @@ -274,3 +276,76 @@ def test_from_spec_refuses_dataclass_not_in_allowlist(): def test_from_spec_rejects_bad_source_type(): with pytest.raises(PretabSerializationError): Preprocessor.from_spec(12345) + + +# --- strict JSON: no NaN / Infinity tokens (issue #80) ----------------------------- + + +def _strict_loads(text): + def reject(token): + raise ValueError(f"non-standard JSON token {token}") + + return json.loads(text, parse_constant=reject) + + +@pytest.fixture +def frame_with_missing(): + rng = np.random.default_rng(0) + a = rng.normal(size=80) + a[:6] = np.nan + X = pd.DataFrame({"a": a, "b": rng.exponential(size=80), "c": rng.choice(["x", "y"], 80)}) + return X, rng.normal(size=80) + + +def test_default_spec_file_is_strict_json(frame_with_missing, tmp_path): + """The imputers' NaN missing-value marker was written as a bare NaN token, + which strict JSON parsers (JavaScript, Go, Rust, ...) reject.""" + X, y = frame_with_missing + pre = Preprocessor(random_state=0).fit(X, y) + path = tmp_path / "spec.json" + pre.to_spec(path) + text = path.read_text(encoding="utf-8") + assert "NaN" not in text and "Infinity" not in text + loaded = Preprocessor.from_spec(_strict_loads(text)) + np.testing.assert_array_equal(np.asarray(loaded.transform(X)), np.asarray(pre.transform(X))) + assert loaded.fingerprint_ == pre.fingerprint_ + + +def test_nan_category_in_an_object_array_is_strict_json(): + """A NaN category of an un-imputed one-hot encoder sits in an object array.""" + rng = np.random.default_rng(0) + city = rng.choice(["x", "y", "z"], 80).astype(object) + city[:5] = np.nan + X = pd.DataFrame({"a": rng.normal(size=80), "city": city}) + pre = Preprocessor(categorical_method="one-hot", categorical_imputation=None, random_state=0).fit(X, X["a"]) + text = json.dumps(pre.to_spec(), allow_nan=False) + assert '{"__float__": "nan"}' in text + loaded = Preprocessor.from_spec(_strict_loads(text)) + np.testing.assert_array_equal(np.asarray(loaded.transform(X)), np.asarray(pre.transform(X))) + assert loaded.fingerprint_ == pre.fingerprint_ + + +def test_non_finite_floats_round_trip(): + from pretab.compose.serialize import _decode, _encode + + array = np.array([[1.0, np.nan], [np.inf, -np.inf]]) + decoded = _decode(_strict_loads(json.dumps(_encode(array), allow_nan=False))) + assert isinstance(decoded, np.ndarray) + np.testing.assert_array_equal(decoded, array) + assert decoded.dtype == array.dtype + nan, neg_inf = ( + float(cast(float, _decode(_strict_loads(json.dumps(_encode(value), allow_nan=False))))) + for value in (np.nan, -np.inf) + ) + assert math.isnan(nan) and neg_inf == -math.inf + + +def test_specs_with_bare_nan_tokens_still_load(frame_with_missing): + X, y = frame_with_missing + pre = Preprocessor(random_state=0).fit(X, y) + spec = json.loads(json.dumps(pre.to_spec())) + # Rewrite the tagged floats the way older versions wrote them: as bare NaN. + legacy_text = json.dumps(spec).replace('{"__float__": "nan"}', "NaN") + assert "NaN" in legacy_text + loaded = Preprocessor.from_spec(json.loads(legacy_text)) + np.testing.assert_array_equal(np.asarray(loaded.transform(X)), np.asarray(pre.transform(X)))