diff --git a/docs/core_concepts/reproducibility.md b/docs/core_concepts/reproducibility.md index 8aa6490..63755dd 100644 --- a/docs/core_concepts/reproducibility.md +++ b/docs/core_concepts/reproducibility.md @@ -29,6 +29,14 @@ directly; they are fit outside `Preprocessor` since they operate on the whole fe reconstructs it. The spec records the schema and library versions, the resolved parameters, and the per-representation fitted state (parameters, knots, centers, columns, scaling). +The spec is strict JSON, so any standard parser (JavaScript, Go, Rust, JSON schema validators) +reads it. Non-finite floats, such as the imputers' `missing_values=np.nan`, are stored as a +tagged value, `{"__float__": "nan"}` (or `"inf"`, `"-inf"`), instead of a bare `NaN` token. +Specs written by PreTab 1.0.0, which contain bare `NaN` tokens, still load. +The reverse is not supported: older readers cannot decode the new float tags. +The tagged encoding also changes fingerprints for fitted state containing non-finite +values, including the default imputers; this change does not alter transformed values. + ```python spec = pre.to_spec() # returns a dict pre.to_spec("representation.json") # or writes JSON to a path diff --git a/pretab/compose/serialize.py b/pretab/compose/serialize.py index 9d9569f..0c7a424 100644 --- a/pretab/compose/serialize.py +++ b/pretab/compose/serialize.py @@ -12,10 +12,16 @@ constructor params, per-representation summary, output column order) plus a ``state`` payload that JSON-encodes the fitted object graph (numpy arrays, knot / center / bin locations, scalers, nested estimators) for exact reconstruction. + +The document is strict (RFC 8259) JSON: non-finite floats, such as an imputer's +``missing_values=np.nan``, are tagged as ``{"__float__": "nan"}`` (``"inf"``, +``"-inf"``), so parsers outside Python read it. Bare ``NaN`` tokens written by +earlier versions are still decoded. """ import dataclasses import importlib +import math from collections import UserList from typing import Any, cast @@ -89,14 +95,23 @@ def _resolve(dotted: str): # --- encoding ------------------------------------------------------------ +def _encode_float(value): + """Return ``value``, tagging NaN / +-inf, which strict JSON cannot represent.""" + if math.isfinite(value): + return value + return {"__float__": repr(float(value))} + + def _encode(obj): if isinstance(obj, np.bool_): return bool(obj) if isinstance(obj, np.integer): return int(obj) if isinstance(obj, np.floating): - return float(obj) - if obj is None or isinstance(obj, (bool, int, float, str)): + return _encode_float(float(obj)) + if isinstance(obj, float): + return _encode_float(obj) + if obj is None or isinstance(obj, (bool, int, str)): return obj if obj is UNSET: return {"__unset__": True} @@ -106,6 +121,8 @@ def _encode(obj): # tolist() keeps the elements of an object array as they are (e.g. # numpy scalars such as np.bool_), so encode each one. data = _map_elements(data, obj.ndim, _encode) + elif obj.dtype.kind == "f" and not np.isfinite(obj).all(): + data = _map_elements(data, obj.ndim, _encode_float) return {"__ndarray__": {"dtype": obj.dtype.str, "shape": list(obj.shape), "data": data}} if isinstance(obj, np.dtype): return {"__npdtype__": obj.str} @@ -152,7 +169,9 @@ def _json_safe(obj): Keeps JSON-native values verbatim and falls back to the tagged :func:`_encode` form only for exotic values. The result is informational and never decoded. """ - if obj is None or isinstance(obj, (bool, int, float, str)): + if isinstance(obj, float): + return _encode_float(obj) + if obj is None or isinstance(obj, (bool, int, str)): return obj if isinstance(obj, dict) and all(isinstance(k, str) for k in obj): return {k: _json_safe(v) for k, v in obj.items()} @@ -174,7 +193,10 @@ def _decode_ndarray(payload: dict) -> np.ndarray: for index, element in enumerate(elements): arr[index] = element return arr.reshape(shape) - arr = np.array(payload["data"], dtype=dtype) + data = payload["data"] + if dtype.kind == "f": + data = _map_elements(data, len(payload["shape"]), _decode) + arr = np.array(data, dtype=dtype) return arr.reshape(payload["shape"]) @@ -188,6 +210,8 @@ def _decode(obj): return _decode_ndarray(obj["__ndarray__"]) if "__unset__" in obj: return UNSET + if "__float__" in obj: + return float(obj["__float__"]) if "__npdtype__" in obj: return np.dtype(obj["__npdtype__"]) if "__type__" in obj: diff --git a/pretab/preprocessor.py b/pretab/preprocessor.py index c6aab66..aaba9df 100644 --- a/pretab/preprocessor.py +++ b/pretab/preprocessor.py @@ -112,9 +112,9 @@ def _preset_numerical_method(task) -> str: def _spec_json(spec: dict, **kwargs) -> str: - """Serialize a spec to JSON text, raising a typed error for unsupported state.""" + """Serialize a spec to strict JSON, raising a typed error for unsupported state.""" try: - return json.dumps(spec, **kwargs) + return json.dumps(spec, allow_nan=False, **kwargs) except (TypeError, ValueError) as exc: raise PretabSerializationError(f"The fitted state cannot be written as JSON: {exc}") from exc @@ -1355,7 +1355,10 @@ def to_spec(self, path=None) -> dict: Parameters ---------- path : str, os.PathLike, or None, default=None - When given, the spec is also written to this path as UTF-8 JSON. + When given, the spec is also written to this path as strict UTF-8 JSON: + non-finite floats are stored as ``{"__float__": "nan"}`` (or ``"inf"`` / + ``"-inf"``), so the file contains no ``NaN`` / ``Infinity`` tokens and + any standard JSON parser reads it. Returns ------- @@ -1387,7 +1390,9 @@ def from_spec(cls, source) -> "Preprocessor": Parameters ---------- source : str, os.PathLike, or dict - A path to a JSON spec file, or the spec dictionary itself. + A path to a JSON spec file, or the spec dictionary itself. Specs written + by PreTab 1.0.0, which store non-finite floats as bare ``NaN`` tokens, + still load. Returns ------- diff --git a/tests/integration/test_serialization.py b/tests/integration/test_serialization.py index 55c9000..b0a4f17 100644 --- a/tests/integration/test_serialization.py +++ b/tests/integration/test_serialization.py @@ -7,6 +7,8 @@ """ import json +import math +from typing import cast import numpy as np import pandas as pd @@ -328,6 +330,124 @@ def test_from_spec_rejects_bad_source_type(): Preprocessor.from_spec(12345) +# --- strict JSON: no NaN / Infinity tokens (issue #80) ----------------------------- + + +def _strict_loads(text): + def reject(token): + raise ValueError(f"non-standard JSON token {token}") + + return json.loads(text, parse_constant=reject) + + +@pytest.fixture +def frame_with_missing(): + rng = np.random.default_rng(0) + a = rng.normal(size=80) + a[:6] = np.nan + X = pd.DataFrame({"a": a, "b": rng.exponential(size=80), "c": rng.choice(["x", "y"], 80)}) + return X, rng.normal(size=80) + + +def test_default_spec_file_is_strict_json(frame_with_missing, tmp_path): + """The imputers' NaN missing-value marker was written as a bare NaN token, + which strict JSON parsers (JavaScript, Go, Rust, ...) reject.""" + X, y = frame_with_missing + pre = Preprocessor(random_state=0).fit(X, y) + path = tmp_path / "spec.json" + fingerprint = pre.fingerprint_ + pre.to_spec(path) + text = path.read_text(encoding="utf-8") + pre.to_spec(path) + assert path.read_text(encoding="utf-8") == text + assert pre.fingerprint_ == fingerprint + assert "NaN" not in text and "Infinity" not in text + loaded = Preprocessor.from_spec(_strict_loads(text)) + np.testing.assert_array_equal(np.asarray(loaded.transform(X)), np.asarray(pre.transform(X))) + assert loaded.fingerprint_ == pre.fingerprint_ + + +def test_nan_category_in_an_object_array_is_strict_json(): + """A NaN category of an un-imputed one-hot encoder sits in an object array.""" + rng = np.random.default_rng(0) + city = rng.choice(["x", "y", "z"], 80).astype(object) + city[:5] = np.nan + X = pd.DataFrame({"a": rng.normal(size=80), "city": city}) + pre = Preprocessor(categorical_method="one-hot", categorical_imputation=None, random_state=0).fit(X, X["a"]) + text = json.dumps(pre.to_spec(), allow_nan=False) + assert '{"__float__": "nan"}' in text + loaded = Preprocessor.from_spec(_strict_loads(text)) + np.testing.assert_array_equal(np.asarray(loaded.transform(X)), np.asarray(pre.transform(X))) + assert loaded.fingerprint_ == pre.fingerprint_ + + +def test_non_finite_floats_round_trip(): + from pretab.compose.serialize import _decode, _encode + + array = np.array([[1.0, np.nan], [np.inf, -np.inf]]) + decoded = _decode(_strict_loads(json.dumps(_encode(array), allow_nan=False))) + assert isinstance(decoded, np.ndarray) + np.testing.assert_array_equal(decoded, array) + assert decoded.dtype == array.dtype + nan, neg_inf = ( + float(cast(float, _decode(_strict_loads(json.dumps(_encode(value), allow_nan=False))))) + for value in (np.nan, -np.inf) + ) + assert math.isnan(nan) and neg_inf == -math.inf + + +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +@pytest.mark.parametrize("shape", [(), (4,), (2, 2), (0, 2)]) +def test_strict_json_float_array_shapes_and_dtypes_round_trip(dtype, shape): + from pretab.compose.serialize import _decode, _encode + + array = np.resize(np.array([np.nan, np.inf, -np.inf, 1.0], dtype=dtype), shape) + decoded = _decode(_strict_loads(json.dumps(_encode(array), allow_nan=False))) + + assert isinstance(decoded, np.ndarray) + assert decoded.shape == shape + assert decoded.dtype == array.dtype + np.testing.assert_array_equal(decoded, array) + + +@pytest.mark.parametrize("shape", [(), (6,), (2, 3), (0, 2)]) +def test_strict_json_object_arrays_preserve_nested_elements(shape): + from pretab.compose.serialize import _decode, _encode + + elements = [ + [np.bool_(True), np.float32(np.nan)], + (np.int64(7), np.float64(np.inf)), + np.array([np.nan, -np.inf], dtype=np.float32), + {"missing": np.float32(np.nan)}, + np.bool_(False), + np.int64(3), + ] + array = np.empty(shape, dtype=object) + for index in range(array.size): + array.flat[index] = elements[index] + decoded = _decode(_strict_loads(json.dumps(_encode(array), allow_nan=False))) + + assert isinstance(decoded, np.ndarray) + assert decoded.shape == shape + assert decoded.dtype == object + for actual, expected in zip(decoded.flat, array.flat, strict=True): + assert type(actual) is type(expected) or isinstance(expected, np.generic) + np.testing.assert_equal(actual, expected) + if isinstance(expected, np.ndarray): + assert actual.dtype == expected.dtype + + +def test_specs_with_bare_nan_tokens_still_load(frame_with_missing): + X, y = frame_with_missing + pre = Preprocessor(random_state=0).fit(X, y) + spec = json.loads(json.dumps(pre.to_spec())) + # Rewrite the tagged floats the way older versions wrote them: as bare NaN. + legacy_text = json.dumps(spec).replace('{"__float__": "nan"}', "NaN") + assert "NaN" in legacy_text + loaded = Preprocessor.from_spec(json.loads(legacy_text)) + np.testing.assert_array_equal(np.asarray(loaded.transform(X)), np.asarray(pre.transform(X))) + + # --- object arrays and builtin dtypes (issue #66) --------------------------------- @@ -362,7 +482,8 @@ def test_builtin_and_numpy_dtypes_round_trip(dtype): assert loaded.fingerprint_ == pre.fingerprint_ -def test_failed_save_leaves_an_existing_spec_untouched(tmp_path, monkeypatch): +@pytest.mark.parametrize("unsupported", [object(), np.nan, np.inf, -np.inf], ids=["object", "nan", "inf", "neg_inf"]) +def test_failed_save_leaves_an_existing_spec_untouched(tmp_path, monkeypatch, unsupported): import pretab.preprocessor as preprocessor_module frame = pd.DataFrame({"age": np.linspace(20.0, 60.0, 30)}) @@ -375,11 +496,13 @@ def test_failed_save_leaves_an_existing_spec_untouched(tmp_path, monkeypatch): # already-opened file used to leave a truncated, invalid spec behind. spec = pre.to_spec() monkeypatch.setattr( - preprocessor_module, "preprocessor_to_spec", lambda _: {**spec, "state": {"unsupported_": object()}} + preprocessor_module, "preprocessor_to_spec", lambda _: {**spec, "state": {"unsupported_": unsupported}} ) with pytest.raises(PretabSerializationError, match="cannot be written as JSON"): pre.to_spec(path) assert path.read_text(encoding="utf-8") == saved + with pytest.raises(PretabSerializationError, match="cannot be written as JSON"): + _ = pre.fingerprint_ def test_types_a_spec_cannot_load_are_rejected_at_save_time():