diff --git a/CHANGELOG.md b/CHANGELOG.md index eabe1a6..1ab0bb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,43 @@ inside each artifact (`schema_version`). ## [Unreleased] +## [0.9.1] - 2026-09-27 + +Missing values are refused instead of mis-compiled. No change to the runtime, +the exporters, the artifact schema, or any artifact built from data without +missing values; the benchmark does not move. + +The histogram backend trains on NaN and learns, at every split, which way +missing values go. It is used whenever monotone constraints are declared +(since 0.2.0) and by `compile_selected` by default. The artifact has no +missing-value branch; it imputes the baseline. So the model and the artifact +scored rows with missing values differently, and nothing said so. On top of +that, `compile_selected` took the baseline as `np.median`, which is NaN for +any column with a missing value. The artifact then carried a NaN baseline, +which is not valid JSON, and sent NaN through tree comparisons, which the +specification forbids (§8). + +### Fixed +- `train_whitebox` refuses NaN in `X` on every backend, naming the columns. + The classic backend already refused it, with scikit-learn's message; the + histogram backend accepted it. The message gives the fix: impute, and add a + 0/1 missing-indicator column, which becomes an ordinary feature with its + own splits and reason code. +- `compile_selected` refuses NaN before the ceiling spends its budget. +- `build_artifact` refuses a non-finite baseline, and a split at a non-finite + threshold (a split at infinity separates missing values from every + observed value). +- Extraction refuses a histogram-boosting model that learned missing-value + routing. Every such split records a direction, learned or not, so the old + note fired on nearly every model and proved nothing; it is gone. Without + NaN at fit time, scikit-learn sends missing values to the larger child, so + a split whose direction contradicts its counts can only have seen NaN, and + that is now an error. +- The duplicate check's message named only one cause, repeat applicants + (`group=`). It now also names rows that are identical by construction + across different entities, such as a no-record code in every feature, + where nothing leaks and the fix is to raise `duplicate_threshold`. + ## [0.9.0] - 2026-09-27 The teacher as yardstick, the target as a selected parameter. diff --git a/CITATION.cff b/CITATION.cff index f3a45dc..0f7fc3c 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -19,7 +19,7 @@ authors: repository-code: "https://github.com/orgoca/CompileML" url: "https://github.com/orgoca/CompileML" license: Apache-2.0 -version: 0.9.0 +version: 0.9.1 date-released: "2026-09-27" keywords: - credit risk diff --git a/docs/faq.md b/docs/faq.md index 9a27ffc..6ee04bf 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -148,6 +148,14 @@ By declared policy inside the artifact: `"baseline"` re-applies the training-time imputation at decision time; `"reject"` refuses the row. NaN never routes silently through a tree comparison, in any runtime. +The artifact has no missing-value branch, so the model it compiles must not +have learned one either. `train_whitebox` and `compile_selected` refuse NaN +in `X`. `build_artifact` refuses a non-finite baseline or split threshold, +and a histogram-boosting model whose splits show it was trained on NaN. +When missingness carries information, impute the column and +add a 0/1 missing-indicator column. The indicator is an ordinary feature, so +it gets its own splits and its own reason code. + ## Is the artifact hash a signature? No — it is an integrity check. Loaders verify it by default and refuse a diff --git a/src/compileml/__init__.py b/src/compileml/__init__.py index 8de878d..dfbc672 100644 --- a/src/compileml/__init__.py +++ b/src/compileml/__init__.py @@ -16,7 +16,7 @@ # this attribute at build time (setuptools dynamic version), so the wheel # metadata, `compileml.__version__`, `compileml inspect`, and the # `compileml_version` recorded inside every artifact can never disagree. -__version__ = "0.9.0" +__version__ = "0.9.1" __all__ = ["compile_selected", "decide", "load_artifact", "verify_artifact", "__version__"] diff --git a/src/compileml/artifact/build.py b/src/compileml/artifact/build.py index 836f4fe..313071b 100644 --- a/src/compileml/artifact/build.py +++ b/src/compileml/artifact/build.py @@ -155,11 +155,28 @@ def build_artifact( base_vals = [float(b) for b in np.asarray(baseline, dtype=float).reshape(-1)] if len(names) != len(base_vals): raise ValueError("feature_names and baseline must have the same length") + bad_base = [names[j] for j, b in enumerate(base_vals) if not np.isfinite(b)] + if bad_base: + raise ValueError( + f"baseline is not finite for {bad_base}. The baseline is the value imputed " + "for missing inputs and the reference point of every attribution, so it must " + "be a real number; np.nanmedian skips missing values when computing medians." + ) # --- extract + quantize ------------------------------------------------- extracted = extract_trees(model) if extracted.n_features and extracted.n_features != len(names): raise ValueError(f"model expects {extracted.n_features} features, got {len(names)} names") + for t_idx, tree in enumerate(extracted.trees): + for f, thr in zip(tree["feature"], tree["threshold"]): + if f >= 0 and not np.isfinite(thr): + raise ValueError( + f"tree {t_idx} splits {names[f]!r} at a non-finite threshold ({thr}). " + "A split at infinity separates missing values from every observed " + "value: the source model learned a missing-value routing the artifact " + "cannot represent. Impute the column, add a 0/1 missing-indicator " + "column, and retrain." + ) if threshold_decimals is not None: # Quantize split thresholds for decimal-arithmetic targets (spec §11). # Every runtime — Python included — then compares identical values; diff --git a/src/compileml/compile/distill.py b/src/compileml/compile/distill.py index bca7a60..91791df 100644 --- a/src/compileml/compile/distill.py +++ b/src/compileml/compile/distill.py @@ -23,6 +23,30 @@ from compileml.compile.monotone import normalize_constraints +def refuse_missing_values(X_arr: np.ndarray, feature_names=None) -> None: + """Raise if ``X_arr`` holds NaN. + + The histogram backend trains on NaN and learns a direction for missing + values at every split, but the compiled artifact has no missing-value + branch: it imputes the baseline instead. The model and the artifact + would then score the same row differently, silently. The classic backend + already refused NaN; this makes both refuse it, with the fix in the + message. + """ + missing = np.isnan(X_arr) + if not missing.any(): + return + cols = np.flatnonzero(missing.any(axis=0)) + named = [str(feature_names[j]) if feature_names is not None else int(j) for j in cols] + shown = named[:10] + (["..."] if len(named) > 10 else []) + raise ValueError( + f"X has missing values (NaN) in {len(cols)} column(s): {shown}. The compiled " + "artifact has no missing-value branch, so a whitebox trained on NaN would score " + "missing values one way and the artifact another. Impute these columns and add a " + "0/1 missing-indicator column for each before training." + ) + + def train_whitebox( X, target=None, @@ -73,6 +97,9 @@ def train_whitebox( outcomes. The returned fidelity metrics stay unweighted. Weighting cannot create an effect depth 2 cannot express — a segment-only interaction is three-way — see the tuning guide. + + Missing values (NaN) in ``X`` are refused on every backend; see + :func:`refuse_missing_values`. """ if teacher_latent is not None: if target is not None: @@ -96,6 +123,8 @@ def train_whitebox( ) X_arr = np.asarray(X, dtype=float) + feature_names = list(X.columns) if hasattr(X, "columns") else None + refuse_missing_values(X_arr, feature_names) y = np.asarray(target, dtype=float).reshape(-1) weights = None if sample_weight is not None: @@ -104,7 +133,6 @@ def train_whitebox( raise ValueError("sample_weight must have one weight per row") if (weights < 0).any() or not np.isfinite(weights).all() or weights.sum() <= 0: raise ValueError("sample_weight must be finite, non-negative, and not all zero") - feature_names = list(X.columns) if hasattr(X, "columns") else None cst = normalize_constraints(monotone_constraints, X_arr.shape[1], feature_names=feature_names) if backend not in (None, "hist", "gbr"): raise ValueError("backend must be None, 'hist' or 'gbr'") diff --git a/src/compileml/compile/extract.py b/src/compileml/compile/extract.py index bcc8b66..e2543d5 100644 --- a/src/compileml/compile/extract.py +++ b/src/compileml/compile/extract.py @@ -160,6 +160,11 @@ def _extract_sklearn_hist(model) -> ExtractedModel: drift). Leaf values arrive pre-shrunk, so ``learning_rate`` is 1.0; the base is ``_baseline_prediction``. Thresholds are float64 and the split convention is ``x <= threshold -> left``, matching ours. + + Every HGB split records a missing-value direction, learned or not, so + the flag alone proves nothing. A model trained on NaN is refused when a + split's direction contradicts sklearn's default (the larger child) or + its threshold is infinite; either can only come from NaN at fit time. """ n_features = int(model.n_features_in_) is_cat = getattr(model, "is_categorical_", None) @@ -172,7 +177,7 @@ def _extract_sklearn_hist(model) -> ExtractedModel: "encode categoricals numerically before distilling" ) trees = [] - missing_right_nodes = 0 + learned_missing: dict[int, int] = {} for predictors in model._predictors: if len(predictors) != 1: raise ValueError("multi-output HistGradientBoosting models are not supported") @@ -197,8 +202,14 @@ def _extract_sklearn_hist(model) -> ExtractedModel: threshold[i] = float(node["num_threshold"]) left[i] = int(node["left"]) right[i] = int(node["right"]) - if not bool(node["missing_go_to_left"]): - missing_right_nodes += 1 + # Without NaN at fit time, HGB sends missing values to the child + # with more samples. A split that disagrees with its counts, or + # sits at infinity (missing vs every observed value), saw NaN. + majority_left = nodes[left[i]]["count"] > nodes[right[i]]["count"] + if bool(node["missing_go_to_left"]) != bool(majority_left) or not np.isfinite( + threshold[i] + ): + learned_missing[feature[i]] = learned_missing.get(feature[i], 0) + 1 trees.append( { "feature": feature, @@ -208,17 +219,16 @@ def _extract_sklearn_hist(model) -> ExtractedModel: "value": value, } ) - notes: list[str] = [] - if missing_right_nodes: - # Unlike XGBoost, HGB records a routing direction on every split even - # when training saw no NaN, so this is a note, not a warning: the - # artifact has no missing branch either way — missing_policy governs. - notes.append( - f"{missing_right_nodes} split(s) route missing values right in the source " - "model; the artifact has no missing branch (missing_policy governs)." + if learned_missing: + raise ValueError( + f"the model was trained on missing values: {sum(learned_missing.values())} " + f"split(s) on feature index(es) {sorted(learned_missing)} route NaN the way " + "training learned. The artifact has no missing-value branch — it imputes the " + "baseline — so it would score those rows differently from the model. Impute " + "these columns, add a 0/1 missing-indicator column for each, and retrain." ) base = float(np.ravel(model._baseline_prediction)[0]) - return ExtractedModel(trees, base, 1.0, "sklearn_hist", "float64", n_features, notes) + return ExtractedModel(trees, base, 1.0, "sklearn_hist", "float64", n_features) # --------------------------------------------------------------------------- diff --git a/src/compileml/select.py b/src/compileml/select.py index 5c4bbce..88457fd 100644 --- a/src/compileml/select.py +++ b/src/compileml/select.py @@ -37,6 +37,7 @@ from compileml.bands import monotone_quantile_bands from compileml.batch import score_batch from compileml.compile import extract_trees, train_whitebox +from compileml.compile.distill import refuse_missing_values from compileml.reference import fit_reference from compileml.runtime.io import canonical_hash @@ -663,6 +664,7 @@ def compile_selected( ) if len(names) != X_arr.shape[1]: raise ValueError("feature_names length does not match X") + refuse_missing_values(X_arr, names) # before the ceiling spends its budget notes: list[str] = [] # ---------------------------------------------------------- partitions @@ -693,7 +695,10 @@ def compile_selected( f"{duplicates} rows in Select or Report ({dup_fraction:.1%}) have content " "identical to a " f"row in an earlier partition, above duplicate_threshold={duplicate_threshold}. " - "Repeat applicants? Pass group= so each lands in one partition." + "If the same entity appears more than once, pass group= so each lands in one " + "partition. If different entities share identical rows by construction — a " + "no-record code in every feature, say — nothing leaks: raise duplicate_threshold " + f"above {dup_fraction:.3f}." ) # ------------------------------------------------------------- ceiling diff --git a/tests/test_missing_values.py b/tests/test_missing_values.py new file mode 100644 index 0000000..88ab8aa --- /dev/null +++ b/tests/test_missing_values.py @@ -0,0 +1,121 @@ +"""Missing values are refused, not mis-compiled. + +The artifact has no missing-value branch: it imputes the baseline. A model +that learned where NaN goes would score those rows one way and the artifact +another, so every path that could produce that pairing refuses it. +""" + +import warnings +from dataclasses import replace + +import numpy as np +import pytest +from sklearn.ensemble import HistGradientBoostingRegressor + +from compileml.artifact import build_artifact +from compileml.bands import monotone_quantile_bands +from compileml.compile import extract_trees, train_whitebox +from compileml.select import compile_selected + +N, P = 3000, 4 +NAMES = [f"f{i}" for i in range(P)] + + +@pytest.fixture(scope="module") +def data(): + rng = np.random.default_rng(5) + X = rng.standard_normal((N, P)) + logit = X[:, 0] - 0.6 * X[:, 1] + 0.5 * X[:, 2] * X[:, 3] + y = (rng.random(N) < 1 / (1 + np.exp(-logit))).astype(int) + X_nan = X.copy() + X_nan[rng.random(N) < 0.2, 0] = np.nan + X_nan[rng.random(N) < 0.05, 3] = np.nan + return X, X_nan, y + + +def _build(model, X, y, baseline=None): + latent = np.clip(model.predict(X), 0.0, 1.0) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return build_artifact( + model, + NAMES, + np.median(X, axis=0) if baseline is None else baseline, + monotone_quantile_bands(latent, y, n_bands=5), + calibration_latent=latent, + calibration_y=y, + ) + + +@pytest.mark.parametrize( + "kwargs", + [{}, {"backend": "gbr"}, {"backend": "hist"}, {"monotone_constraints": [1, 0, 0, 0]}], +) +def test_train_whitebox_refuses_nan_on_every_backend(data, kwargs): + _, X_nan, y = data + with pytest.raises(ValueError, match=r"missing values \(NaN\) in 2 column\(s\): \[0, 3\]"): + train_whitebox(X_nan, y.astype(float), n_estimators=5, **kwargs) + + +def test_compile_selected_refuses_nan_before_the_ceiling_runs(data): + _, X_nan, y = data + calls = [] + + def ceiling(Xf, yf): + calls.append(1) + raise AssertionError("the ceiling must not be fitted") + + with pytest.raises(ValueError, match=r"missing values \(NaN\).*\['f0', 'f3'\]"): + compile_selected(X_nan, y, ceiling=ceiling, feature_names=NAMES) + assert calls == [] + + +def test_extraction_refuses_a_model_trained_on_nan(data): + _, X_nan, y = data + model = HistGradientBoostingRegressor( + max_iter=30, max_depth=2, early_stopping=False, random_state=0 + ).fit(X_nan, y.astype(float)) + with pytest.raises(ValueError, match=r"trained on missing values.*\[0"): + extract_trees(model) + + +@pytest.mark.parametrize("seed", range(4)) +def test_extraction_accepts_models_that_never_saw_nan(data, seed): + """HGB records a missing direction on every split; without NaN it is the default.""" + X, _, y = data + weights = np.random.default_rng(seed).random(N) + 0.5 if seed % 2 else None + common = {"max_iter": 40, "max_depth": 2, "early_stopping": False, "random_state": seed} + for model in ( + HistGradientBoostingRegressor(**common), + HistGradientBoostingRegressor(**common, monotonic_cst=[1, -1, 0, 0]), + ): + model.fit(X, y.astype(float), sample_weight=weights) + assert extract_trees(model).trees + + +def test_build_refuses_a_non_finite_baseline(data): + X, _, y = data + model, _ = train_whitebox(X, y.astype(float), n_estimators=10, backend="hist") + for bad in (np.nan, np.inf): + baseline = np.median(X, axis=0) + baseline[2] = bad + with pytest.raises(ValueError, match=r"baseline is not finite for \['f2'\]"): + _build(model, X, y, baseline=baseline) + art = _build(model, X, y) + assert all(np.isfinite(art["features"]["baseline"])) + + +def test_build_refuses_a_non_finite_threshold(data): + X, _, y = data + model, _ = train_whitebox(X, y.astype(float), n_estimators=10, backend="hist") + extracted = extract_trees(model) + tree = dict(extracted.trees[0]) + split = next(i for i, f in enumerate(tree["feature"]) if f >= 0) + tree["threshold"] = list(tree["threshold"]) + tree["threshold"][split] = float("inf") + tampered = replace(extracted, trees=[tree, *extracted.trees[1:]]) + latent = np.clip(model.predict(X), 0.0, 1.0) + with pytest.raises(ValueError, match="non-finite threshold"): + build_artifact( + tampered, NAMES, np.median(X, axis=0), monotone_quantile_bands(latent, y, n_bands=5) + ) diff --git a/tests/test_select.py b/tests/test_select.py index 68ad2c6..671ef07 100644 --- a/tests/test_select.py +++ b/tests/test_select.py @@ -141,7 +141,8 @@ def test_content_duplicates_are_counted_and_thresholded(data): y_dup = y.copy() y_dup[parts.report[:100]] = y[parts.fit[:100]] assert duplicate_rows(row_content_hashes(X_dup, y_dup), parts) == 100 - with pytest.raises(ValueError, match="identical"): + # both causes are named: repeat entities (group=) and identical-by-construction rows + with pytest.raises(ValueError, match=r"identical.*group=.*raise duplicate_threshold above"): compile_selected( X_dup, y_dup,