From 9eda1f9500f9261268fdaea5d3aa90d69204efd3 Mon Sep 17 00:00:00 2001 From: Carlos Ortiz Date: Sun, 27 Sep 2026 20:33:05 -0600 Subject: [PATCH 1/2] Refuse missing values instead of compiling a model the artifact cannot reproduce The histogram backend trains on NaN and learns which way missing values go at every split; the artifact has no missing-value branch and imputes the baseline instead, so the two scored those rows differently without a word. compile_selected also took the baseline as np.median, which is NaN for any column with a missing value, and shipped it: invalid JSON, and NaN routed through tree comparisons, which spec section 8 forbids. train_whitebox and compile_selected now refuse NaN in X, naming the columns and the fix (impute, add a 0/1 missing-indicator column). build_artifact refuses a non-finite baseline or split threshold. Extraction refuses a histogram-boosting model whose splits show it saw NaN: without NaN at fit time scikit-learn sends missing values to the larger child, so a direction that contradicts the child counts, or a split at infinity, can only come from training on NaN. The old note fired on nearly every model and proved nothing, so it is gone. Checked on 30 models trained without NaN (weighted, monotone, early stopping): no false positives. The duplicate check's message now names both causes: repeat entities (group=) and rows identical by construction (raise duplicate_threshold). --- docs/faq.md | 8 ++ src/compileml/artifact/build.py | 17 +++++ src/compileml/compile/distill.py | 30 +++++++- src/compileml/compile/extract.py | 34 ++++++--- src/compileml/select.py | 7 +- tests/test_missing_values.py | 121 +++++++++++++++++++++++++++++++ tests/test_select.py | 3 +- 7 files changed, 205 insertions(+), 15 deletions(-) create mode 100644 tests/test_missing_values.py diff --git a/docs/faq.md b/docs/faq.md index 9a27ffc..6ee04bf 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -148,6 +148,14 @@ By declared policy inside the artifact: `"baseline"` re-applies the training-time imputation at decision time; `"reject"` refuses the row. NaN never routes silently through a tree comparison, in any runtime. +The artifact has no missing-value branch, so the model it compiles must not +have learned one either. `train_whitebox` and `compile_selected` refuse NaN +in `X`. `build_artifact` refuses a non-finite baseline or split threshold, +and a histogram-boosting model whose splits show it was trained on NaN. +When missingness carries information, impute the column and +add a 0/1 missing-indicator column. The indicator is an ordinary feature, so +it gets its own splits and its own reason code. + ## Is the artifact hash a signature? No — it is an integrity check. Loaders verify it by default and refuse a diff --git a/src/compileml/artifact/build.py b/src/compileml/artifact/build.py index 836f4fe..313071b 100644 --- a/src/compileml/artifact/build.py +++ b/src/compileml/artifact/build.py @@ -155,11 +155,28 @@ def build_artifact( base_vals = [float(b) for b in np.asarray(baseline, dtype=float).reshape(-1)] if len(names) != len(base_vals): raise ValueError("feature_names and baseline must have the same length") + bad_base = [names[j] for j, b in enumerate(base_vals) if not np.isfinite(b)] + if bad_base: + raise ValueError( + f"baseline is not finite for {bad_base}. The baseline is the value imputed " + "for missing inputs and the reference point of every attribution, so it must " + "be a real number; np.nanmedian skips missing values when computing medians." + ) # --- extract + quantize ------------------------------------------------- extracted = extract_trees(model) if extracted.n_features and extracted.n_features != len(names): raise ValueError(f"model expects {extracted.n_features} features, got {len(names)} names") + for t_idx, tree in enumerate(extracted.trees): + for f, thr in zip(tree["feature"], tree["threshold"]): + if f >= 0 and not np.isfinite(thr): + raise ValueError( + f"tree {t_idx} splits {names[f]!r} at a non-finite threshold ({thr}). " + "A split at infinity separates missing values from every observed " + "value: the source model learned a missing-value routing the artifact " + "cannot represent. Impute the column, add a 0/1 missing-indicator " + "column, and retrain." + ) if threshold_decimals is not None: # Quantize split thresholds for decimal-arithmetic targets (spec §11). # Every runtime — Python included — then compares identical values; diff --git a/src/compileml/compile/distill.py b/src/compileml/compile/distill.py index bca7a60..91791df 100644 --- a/src/compileml/compile/distill.py +++ b/src/compileml/compile/distill.py @@ -23,6 +23,30 @@ from compileml.compile.monotone import normalize_constraints +def refuse_missing_values(X_arr: np.ndarray, feature_names=None) -> None: + """Raise if ``X_arr`` holds NaN. + + The histogram backend trains on NaN and learns a direction for missing + values at every split, but the compiled artifact has no missing-value + branch: it imputes the baseline instead. The model and the artifact + would then score the same row differently, silently. The classic backend + already refused NaN; this makes both refuse it, with the fix in the + message. + """ + missing = np.isnan(X_arr) + if not missing.any(): + return + cols = np.flatnonzero(missing.any(axis=0)) + named = [str(feature_names[j]) if feature_names is not None else int(j) for j in cols] + shown = named[:10] + (["..."] if len(named) > 10 else []) + raise ValueError( + f"X has missing values (NaN) in {len(cols)} column(s): {shown}. The compiled " + "artifact has no missing-value branch, so a whitebox trained on NaN would score " + "missing values one way and the artifact another. Impute these columns and add a " + "0/1 missing-indicator column for each before training." + ) + + def train_whitebox( X, target=None, @@ -73,6 +97,9 @@ def train_whitebox( outcomes. The returned fidelity metrics stay unweighted. Weighting cannot create an effect depth 2 cannot express — a segment-only interaction is three-way — see the tuning guide. + + Missing values (NaN) in ``X`` are refused on every backend; see + :func:`refuse_missing_values`. """ if teacher_latent is not None: if target is not None: @@ -96,6 +123,8 @@ def train_whitebox( ) X_arr = np.asarray(X, dtype=float) + feature_names = list(X.columns) if hasattr(X, "columns") else None + refuse_missing_values(X_arr, feature_names) y = np.asarray(target, dtype=float).reshape(-1) weights = None if sample_weight is not None: @@ -104,7 +133,6 @@ def train_whitebox( raise ValueError("sample_weight must have one weight per row") if (weights < 0).any() or not np.isfinite(weights).all() or weights.sum() <= 0: raise ValueError("sample_weight must be finite, non-negative, and not all zero") - feature_names = list(X.columns) if hasattr(X, "columns") else None cst = normalize_constraints(monotone_constraints, X_arr.shape[1], feature_names=feature_names) if backend not in (None, "hist", "gbr"): raise ValueError("backend must be None, 'hist' or 'gbr'") diff --git a/src/compileml/compile/extract.py b/src/compileml/compile/extract.py index bcc8b66..e2543d5 100644 --- a/src/compileml/compile/extract.py +++ b/src/compileml/compile/extract.py @@ -160,6 +160,11 @@ def _extract_sklearn_hist(model) -> ExtractedModel: drift). Leaf values arrive pre-shrunk, so ``learning_rate`` is 1.0; the base is ``_baseline_prediction``. Thresholds are float64 and the split convention is ``x <= threshold -> left``, matching ours. + + Every HGB split records a missing-value direction, learned or not, so + the flag alone proves nothing. A model trained on NaN is refused when a + split's direction contradicts sklearn's default (the larger child) or + its threshold is infinite; either can only come from NaN at fit time. """ n_features = int(model.n_features_in_) is_cat = getattr(model, "is_categorical_", None) @@ -172,7 +177,7 @@ def _extract_sklearn_hist(model) -> ExtractedModel: "encode categoricals numerically before distilling" ) trees = [] - missing_right_nodes = 0 + learned_missing: dict[int, int] = {} for predictors in model._predictors: if len(predictors) != 1: raise ValueError("multi-output HistGradientBoosting models are not supported") @@ -197,8 +202,14 @@ def _extract_sklearn_hist(model) -> ExtractedModel: threshold[i] = float(node["num_threshold"]) left[i] = int(node["left"]) right[i] = int(node["right"]) - if not bool(node["missing_go_to_left"]): - missing_right_nodes += 1 + # Without NaN at fit time, HGB sends missing values to the child + # with more samples. A split that disagrees with its counts, or + # sits at infinity (missing vs every observed value), saw NaN. + majority_left = nodes[left[i]]["count"] > nodes[right[i]]["count"] + if bool(node["missing_go_to_left"]) != bool(majority_left) or not np.isfinite( + threshold[i] + ): + learned_missing[feature[i]] = learned_missing.get(feature[i], 0) + 1 trees.append( { "feature": feature, @@ -208,17 +219,16 @@ def _extract_sklearn_hist(model) -> ExtractedModel: "value": value, } ) - notes: list[str] = [] - if missing_right_nodes: - # Unlike XGBoost, HGB records a routing direction on every split even - # when training saw no NaN, so this is a note, not a warning: the - # artifact has no missing branch either way — missing_policy governs. - notes.append( - f"{missing_right_nodes} split(s) route missing values right in the source " - "model; the artifact has no missing branch (missing_policy governs)." + if learned_missing: + raise ValueError( + f"the model was trained on missing values: {sum(learned_missing.values())} " + f"split(s) on feature index(es) {sorted(learned_missing)} route NaN the way " + "training learned. The artifact has no missing-value branch — it imputes the " + "baseline — so it would score those rows differently from the model. Impute " + "these columns, add a 0/1 missing-indicator column for each, and retrain." ) base = float(np.ravel(model._baseline_prediction)[0]) - return ExtractedModel(trees, base, 1.0, "sklearn_hist", "float64", n_features, notes) + return ExtractedModel(trees, base, 1.0, "sklearn_hist", "float64", n_features) # --------------------------------------------------------------------------- diff --git a/src/compileml/select.py b/src/compileml/select.py index 5c4bbce..88457fd 100644 --- a/src/compileml/select.py +++ b/src/compileml/select.py @@ -37,6 +37,7 @@ from compileml.bands import monotone_quantile_bands from compileml.batch import score_batch from compileml.compile import extract_trees, train_whitebox +from compileml.compile.distill import refuse_missing_values from compileml.reference import fit_reference from compileml.runtime.io import canonical_hash @@ -663,6 +664,7 @@ def compile_selected( ) if len(names) != X_arr.shape[1]: raise ValueError("feature_names length does not match X") + refuse_missing_values(X_arr, names) # before the ceiling spends its budget notes: list[str] = [] # ---------------------------------------------------------- partitions @@ -693,7 +695,10 @@ def compile_selected( f"{duplicates} rows in Select or Report ({dup_fraction:.1%}) have content " "identical to a " f"row in an earlier partition, above duplicate_threshold={duplicate_threshold}. " - "Repeat applicants? Pass group= so each lands in one partition." + "If the same entity appears more than once, pass group= so each lands in one " + "partition. If different entities share identical rows by construction — a " + "no-record code in every feature, say — nothing leaks: raise duplicate_threshold " + f"above {dup_fraction:.3f}." ) # ------------------------------------------------------------- ceiling diff --git a/tests/test_missing_values.py b/tests/test_missing_values.py new file mode 100644 index 0000000..88ab8aa --- /dev/null +++ b/tests/test_missing_values.py @@ -0,0 +1,121 @@ +"""Missing values are refused, not mis-compiled. + +The artifact has no missing-value branch: it imputes the baseline. A model +that learned where NaN goes would score those rows one way and the artifact +another, so every path that could produce that pairing refuses it. +""" + +import warnings +from dataclasses import replace + +import numpy as np +import pytest +from sklearn.ensemble import HistGradientBoostingRegressor + +from compileml.artifact import build_artifact +from compileml.bands import monotone_quantile_bands +from compileml.compile import extract_trees, train_whitebox +from compileml.select import compile_selected + +N, P = 3000, 4 +NAMES = [f"f{i}" for i in range(P)] + + +@pytest.fixture(scope="module") +def data(): + rng = np.random.default_rng(5) + X = rng.standard_normal((N, P)) + logit = X[:, 0] - 0.6 * X[:, 1] + 0.5 * X[:, 2] * X[:, 3] + y = (rng.random(N) < 1 / (1 + np.exp(-logit))).astype(int) + X_nan = X.copy() + X_nan[rng.random(N) < 0.2, 0] = np.nan + X_nan[rng.random(N) < 0.05, 3] = np.nan + return X, X_nan, y + + +def _build(model, X, y, baseline=None): + latent = np.clip(model.predict(X), 0.0, 1.0) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return build_artifact( + model, + NAMES, + np.median(X, axis=0) if baseline is None else baseline, + monotone_quantile_bands(latent, y, n_bands=5), + calibration_latent=latent, + calibration_y=y, + ) + + +@pytest.mark.parametrize( + "kwargs", + [{}, {"backend": "gbr"}, {"backend": "hist"}, {"monotone_constraints": [1, 0, 0, 0]}], +) +def test_train_whitebox_refuses_nan_on_every_backend(data, kwargs): + _, X_nan, y = data + with pytest.raises(ValueError, match=r"missing values \(NaN\) in 2 column\(s\): \[0, 3\]"): + train_whitebox(X_nan, y.astype(float), n_estimators=5, **kwargs) + + +def test_compile_selected_refuses_nan_before_the_ceiling_runs(data): + _, X_nan, y = data + calls = [] + + def ceiling(Xf, yf): + calls.append(1) + raise AssertionError("the ceiling must not be fitted") + + with pytest.raises(ValueError, match=r"missing values \(NaN\).*\['f0', 'f3'\]"): + compile_selected(X_nan, y, ceiling=ceiling, feature_names=NAMES) + assert calls == [] + + +def test_extraction_refuses_a_model_trained_on_nan(data): + _, X_nan, y = data + model = HistGradientBoostingRegressor( + max_iter=30, max_depth=2, early_stopping=False, random_state=0 + ).fit(X_nan, y.astype(float)) + with pytest.raises(ValueError, match=r"trained on missing values.*\[0"): + extract_trees(model) + + +@pytest.mark.parametrize("seed", range(4)) +def test_extraction_accepts_models_that_never_saw_nan(data, seed): + """HGB records a missing direction on every split; without NaN it is the default.""" + X, _, y = data + weights = np.random.default_rng(seed).random(N) + 0.5 if seed % 2 else None + common = {"max_iter": 40, "max_depth": 2, "early_stopping": False, "random_state": seed} + for model in ( + HistGradientBoostingRegressor(**common), + HistGradientBoostingRegressor(**common, monotonic_cst=[1, -1, 0, 0]), + ): + model.fit(X, y.astype(float), sample_weight=weights) + assert extract_trees(model).trees + + +def test_build_refuses_a_non_finite_baseline(data): + X, _, y = data + model, _ = train_whitebox(X, y.astype(float), n_estimators=10, backend="hist") + for bad in (np.nan, np.inf): + baseline = np.median(X, axis=0) + baseline[2] = bad + with pytest.raises(ValueError, match=r"baseline is not finite for \['f2'\]"): + _build(model, X, y, baseline=baseline) + art = _build(model, X, y) + assert all(np.isfinite(art["features"]["baseline"])) + + +def test_build_refuses_a_non_finite_threshold(data): + X, _, y = data + model, _ = train_whitebox(X, y.astype(float), n_estimators=10, backend="hist") + extracted = extract_trees(model) + tree = dict(extracted.trees[0]) + split = next(i for i, f in enumerate(tree["feature"]) if f >= 0) + tree["threshold"] = list(tree["threshold"]) + tree["threshold"][split] = float("inf") + tampered = replace(extracted, trees=[tree, *extracted.trees[1:]]) + latent = np.clip(model.predict(X), 0.0, 1.0) + with pytest.raises(ValueError, match="non-finite threshold"): + build_artifact( + tampered, NAMES, np.median(X, axis=0), monotone_quantile_bands(latent, y, n_bands=5) + ) diff --git a/tests/test_select.py b/tests/test_select.py index 68ad2c6..671ef07 100644 --- a/tests/test_select.py +++ b/tests/test_select.py @@ -141,7 +141,8 @@ def test_content_duplicates_are_counted_and_thresholded(data): y_dup = y.copy() y_dup[parts.report[:100]] = y[parts.fit[:100]] assert duplicate_rows(row_content_hashes(X_dup, y_dup), parts) == 100 - with pytest.raises(ValueError, match="identical"): + # both causes are named: repeat entities (group=) and identical-by-construction rows + with pytest.raises(ValueError, match=r"identical.*group=.*raise duplicate_threshold above"): compile_selected( X_dup, y_dup, From c73a9e73b456b527f488ae16c90ce7b94dde56c8 Mon Sep 17 00:00:00 2001 From: Carlos Ortiz Date: Sun, 27 Sep 2026 20:33:05 -0600 Subject: [PATCH 2/2] release: v0.9.1 Missing values are refused instead of mis-compiled. A patch release: no change to the runtime, the exporters, the artifact schema, or any artifact built from data without missing values. The benchmark does not move, so results.json still records the 0.9.0 run that produced it. --- CHANGELOG.md | 37 +++++++++++++++++++++++++++++++++++++ CITATION.cff | 2 +- src/compileml/__init__.py | 2 +- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eabe1a6..1ab0bb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,43 @@ inside each artifact (`schema_version`). ## [Unreleased] +## [0.9.1] - 2026-09-27 + +Missing values are refused instead of mis-compiled. No change to the runtime, +the exporters, the artifact schema, or any artifact built from data without +missing values; the benchmark does not move. + +The histogram backend trains on NaN and learns, at every split, which way +missing values go. It is used whenever monotone constraints are declared +(since 0.2.0) and by `compile_selected` by default. The artifact has no +missing-value branch; it imputes the baseline. So the model and the artifact +scored rows with missing values differently, and nothing said so. On top of +that, `compile_selected` took the baseline as `np.median`, which is NaN for +any column with a missing value. The artifact then carried a NaN baseline, +which is not valid JSON, and sent NaN through tree comparisons, which the +specification forbids (§8). + +### Fixed +- `train_whitebox` refuses NaN in `X` on every backend, naming the columns. + The classic backend already refused it, with scikit-learn's message; the + histogram backend accepted it. The message gives the fix: impute, and add a + 0/1 missing-indicator column, which becomes an ordinary feature with its + own splits and reason code. +- `compile_selected` refuses NaN before the ceiling spends its budget. +- `build_artifact` refuses a non-finite baseline, and a split at a non-finite + threshold (a split at infinity separates missing values from every + observed value). +- Extraction refuses a histogram-boosting model that learned missing-value + routing. Every such split records a direction, learned or not, so the old + note fired on nearly every model and proved nothing; it is gone. Without + NaN at fit time, scikit-learn sends missing values to the larger child, so + a split whose direction contradicts its counts can only have seen NaN, and + that is now an error. +- The duplicate check's message named only one cause, repeat applicants + (`group=`). It now also names rows that are identical by construction + across different entities, such as a no-record code in every feature, + where nothing leaks and the fix is to raise `duplicate_threshold`. + ## [0.9.0] - 2026-09-27 The teacher as yardstick, the target as a selected parameter. diff --git a/CITATION.cff b/CITATION.cff index f3a45dc..0f7fc3c 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -19,7 +19,7 @@ authors: repository-code: "https://github.com/orgoca/CompileML" url: "https://github.com/orgoca/CompileML" license: Apache-2.0 -version: 0.9.0 +version: 0.9.1 date-released: "2026-09-27" keywords: - credit risk diff --git a/src/compileml/__init__.py b/src/compileml/__init__.py index 8de878d..dfbc672 100644 --- a/src/compileml/__init__.py +++ b/src/compileml/__init__.py @@ -16,7 +16,7 @@ # this attribute at build time (setuptools dynamic version), so the wheel # metadata, `compileml.__version__`, `compileml inspect`, and the # `compileml_version` recorded inside every artifact can never disagree. -__version__ = "0.9.0" +__version__ = "0.9.1" __all__ = ["compile_selected", "decide", "load_artifact", "verify_artifact", "__version__"]