From 56235f220aa70cd3e50fbb8f1b64a33f0988fa87 Mon Sep 17 00:00:00 2001 From: Carlos Ortiz Date: Sun, 27 Sep 2026 10:08:37 -0600 Subject: [PATCH 1/5] Select the target and the configuration on data the report never sees compile_selected implements the protocol of #75. Rows are split into Fit, Select and Report: stratified, out-of-time by date, or by group so one applicant lands in one partition. The ceiling (the teacher) is tuned once by the user's factory, frozen, and refit K times for cross-fitted soft targets; one search plus K fits, never K searches. Every (alpha, trees, depth, monotone) configuration is compiled on Fit and scored on Select as an integer artifact, so quantization and banding are priced too. The simplest configuration statistically tied with the best wins, preferring alpha = 1 as a governance prior recorded as the tie rule; the floor gates; the winner is refit on Fit union Select; result.report() reads Report once and writes the figures into a hash-covered provenance block, so what ships carries its own reported cost. Below a configurable event count in Select, nested cross-validation takes over with a warning. Whether the whitebox learns from the ceiling's probabilities or from the labels is now a selected parameter. Every step is also a plain function. compileml.batch.score_batch is the runtime's score path over a matrix in NumPy, asserted bit-identical to decide() on every row across calibration modes, input precision and missing values; 200,000 rows in under a second. Selection ranks compiled artifacts with it and never computes attribution. The trees axis is free: extract_trees passes an ExtractedModel through, so one fit at the largest tree count is extracted once and compiled from prefixes, verified equal to a separate fit for both backends. train_whitebox(backend="hist") selects the histogram backend explicitly, measured at 1-2 s where the classic backend took 18 s at 30k rows and 293 s at 300k; compile_selected uses it throughout, and the default is unchanged. Validation check 11, selection hygiene, reads the provenance block and lists broken protocol: soft targets not cross-fitted, partitions that overlap, Report read more than once. Advisory unless required, following check 10. sweep_whitebox warns when alpha_grid is not passed and when one holdout is used both to choose and to report. Part of #75; the documentation and the benchmark rerun follow separately. --- CHANGELOG.md | 35 + src/compileml/__init__.py | 13 +- src/compileml/batch.py | 122 ++++ src/compileml/cli.py | 8 +- src/compileml/compile/distill.py | 15 +- src/compileml/compile/extract.py | 10 +- src/compileml/select.py | 978 ++++++++++++++++++++++++++++ src/compileml/tune/sweeps.py | 20 +- src/compileml/validate/framework.py | 71 ++ tests/test_batch.py | 64 ++ tests/test_select.py | 447 +++++++++++++ tests/test_validate.py | 4 +- 12 files changed, 1781 insertions(+), 6 deletions(-) create mode 100644 src/compileml/batch.py create mode 100644 src/compileml/select.py create mode 100644 tests/test_batch.py create mode 100644 tests/test_select.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a434b2..7207a1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,41 @@ inside each artifact (`schema_version`). ## [Unreleased] +### Added +- `compile_selected` ([#75](https://github.com/orgoca/CompileML/issues/75)): + the target and the configuration are chosen on data the report never sees. + Rows are split into Fit, Select and Report; the ceiling (the teacher) is + tuned once, frozen and refit K times for cross-fitted soft targets; every + (α, trees, depth, monotone) configuration is compiled on Fit and scored on + Select as an integer artifact; the simplest configuration statistically + tied with the best wins, preferring α = 1 as a governance prior; the floor + gates; the winner is refit on Fit ∪ Select; and `result.report()` reads + Report once, writing the figures into a hash-covered provenance block. Below + a configurable event count in Select, nested cross-validation takes over. + Whether the whitebox learns from the ceiling's probabilities or from the + labels is now a selected parameter, not an assumption. +- `compileml.batch.score_batch`: the runtime's score path over a matrix, in + NumPy, asserted bit-identical to `decide()` on every row — 200,000 rows in + under a second. Selection ranks compiled artifacts with it; deployments + still use the runtime or an export. +- Validation check 11, selection hygiene: reads the provenance block and + lists broken protocol — soft targets not cross-fitted, partitions that + overlap, Report read more than once. Advisory unless + `require_selection_hygiene=True`, following check 10. +- `train_whitebox(backend="hist")` selects the histogram backend explicitly. + It was measured at 1–2 s where the classic backend took 18 s at 30k rows and + 293 s at 300k; `compile_selected` uses it throughout. The default is + unchanged, since changing it would change every artifact built with the + defaults. +- `extract_trees` passes an `ExtractedModel` through, so a boosted fit can be + extracted once and compiled from prefixes of its trees: the first k trees of + a fit are the k-tree model, verified for both backends. + +### Changed +- `sweep_whitebox` warns when `alpha_grid` is not passed — its default of + pure distillation is the assumption real data falsified — and when `X_val` + is used both to choose and to report. The defaults switch in 1.0. + ## [0.8.0] - 2026-09-13 What compilation cost where decisions are made — and what to do about it. diff --git a/src/compileml/__init__.py b/src/compileml/__init__.py index 52b5774..4673c6d 100644 --- a/src/compileml/__init__.py +++ b/src/compileml/__init__.py @@ -18,4 +18,15 @@ # `compileml_version` recorded inside every artifact can never disagree. __version__ = "0.8.0" -__all__ = ["decide", "load_artifact", "verify_artifact", "__version__"] +__all__ = ["compile_selected", "decide", "load_artifact", "verify_artifact", "__version__"] + + +def __getattr__(name): + # compile_selected is learning-side (NumPy, scikit-learn). Importing it + # lazily keeps `import compileml` free of those dependencies, so the + # runtime story - copy compileml/runtime anywhere - stays true of the root. + if name == "compile_selected": + from compileml.select import compile_selected + + return compile_selected + raise AttributeError(f"module 'compileml' has no attribute {name!r}") diff --git a/src/compileml/batch.py b/src/compileml/batch.py new file mode 100644 index 0000000..0a3fa09 --- /dev/null +++ b/src/compileml/batch.py @@ -0,0 +1,122 @@ +"""Score many rows through an artifact at once, with NumPy. + +The runtime scores one row at a time in the standard library, which is what +lets it run anywhere. Selection scores forty compiled candidates on hundreds +of thousands of rows, where a Python loop costs minutes per candidate. This +module does the same arithmetic vectorised — the same integer sums, the same +float64 comparison, the same rounding — and the test suite asserts it agrees +with :func:`compileml.runtime.decide` on every row. + +Learning side: it reads an artifact and never makes a decision anyone acts +on. Deployments use the runtime or an export. +""" + +from __future__ import annotations + +import numpy as np + +from compileml.runtime.calibrate import PD_SCALE +from compileml.runtime.io import ArtifactError +from compileml.runtime.score import LEAF + + +def _div_rha_nonneg(num: np.ndarray, den) -> np.ndarray: + """Spec §2.2 for non-negative numerators, in int64.""" + den = np.asarray(den, dtype=np.int64) + return (2 * num + den) // (2 * den) + + +def _prepare(artifact: dict, X) -> np.ndarray: + """The runtime's ``_prepare_row`` over a matrix (spec §4, §8).""" + names = artifact["features"]["names"] + X_arr = np.array(X, dtype=np.float64, copy=True) + if X_arr.ndim != 2 or X_arr.shape[1] != len(names): + raise ArtifactError(f"expected {len(names)} feature columns, got shape {X_arr.shape}") + missing = np.isnan(X_arr) + if missing.any(): + if artifact["features"].get("missing_policy", "baseline") == "reject": + rows = int(missing.any(axis=1).sum()) + raise ArtifactError(f"{rows} row(s) have missing values and missing_policy is 'reject'") + baseline = np.asarray(artifact["features"]["baseline"], dtype=np.float64) + X_arr = np.where(missing, baseline[None, :], X_arr) + if artifact["model"].get("input_precision", "float64") == "float32": + X_arr = X_arr.astype(np.float32).astype(np.float64) + return X_arr + + +def raw_micro_batch(model: dict, X: np.ndarray) -> np.ndarray: + """``score_micro`` for every row: integer sums of the leaves each row reaches.""" + n = X.shape[0] + acc = np.full(n, int(model["base_micro"]), dtype=np.int64) + rows = np.arange(n) + for tree in model["trees"]: + feature = np.asarray(tree["feature"], dtype=np.int64) + threshold = np.asarray(tree["threshold"], dtype=np.float64) + left = np.asarray(tree["left"], dtype=np.int64) + right = np.asarray(tree["right"], dtype=np.int64) + value = np.asarray(tree["value_micro"], dtype=np.int64) + node = np.zeros(n, dtype=np.int64) + active = feature[node] != LEAF + while active.any(): + idx = rows[active] + at = node[idx] + go_left = X[idx, feature[at]] <= threshold[at] + node[idx] = np.where(go_left, left[at], right[at]) + active = feature[node] != LEAF + acc += value[node] + return acc + + +def calibrate_ppm_batch( + latent_micro: np.ndarray, calibration: dict | None, micro_scale: int +) -> np.ndarray: + """``calibrate_ppm`` for every row (spec §6).""" + if not calibration: + return _div_rha_nonneg(latent_micro * PD_SCALE, micro_scale) + f = np.asarray(calibration["f_micro"], dtype=np.int64) + pd = np.asarray(calibration["pd_ppm"], dtype=np.int64) + k = len(f) + out = np.empty_like(latent_micro) + low = latent_micro <= f[0] + high = latent_micro >= f[k - 1] + out[low] = pd[0] + out[high] = pd[k - 1] + mid = ~(low | high) + if mid.any(): + lm = latent_micro[mid] + hi = np.searchsorted(f, lm, side="right") + lo = hi - 1 + if calibration.get("mode", "linear_int") == "step": + out[mid] = pd[lo] + else: + num = (pd[hi] - pd[lo]) * (lm - f[lo]) + out[mid] = pd[lo] + _div_rha_nonneg(num, f[hi] - f[lo]) + return out + + +def score_batch(artifact: dict, X) -> dict[str, np.ndarray]: + """Score a matrix of rows: the runtime's score path, one array per output. + + Returns ``raw_micro``, ``latent_micro``, ``latent_int``, ``band_idx`` and + ``pd_ppm`` as int64 arrays, plus ``pd`` as float64. Every integer equals + what ``decide(artifact, row, explain=False)`` returns for that row. + """ + model = artifact["model"] + micro_scale = int(model["micro_scale"]) + ratio = micro_scale // int(artifact["scale"]) + X_arr = _prepare(artifact, X) + + raw = raw_micro_batch(model, X_arr) + latent_micro = np.clip(raw, 0, micro_scale) + latent_int = _div_rha_nonneg(latent_micro, ratio) + edges = np.asarray(artifact["bands"]["edges_int"], dtype=np.int64)[1:-1] + band_idx = np.searchsorted(edges, latent_int, side="right") + pd_ppm = calibrate_ppm_batch(latent_micro, artifact.get("calibration"), micro_scale) + return { + "raw_micro": raw, + "latent_micro": latent_micro, + "latent_int": latent_int, + "band_idx": band_idx, + "pd_ppm": pd_ppm, + "pd": pd_ppm / PD_SCALE, + } diff --git a/src/compileml/cli.py b/src/compileml/cli.py index da9ebf9..b2d12ae 100644 --- a/src/compileml/cli.py +++ b/src/compileml/cli.py @@ -175,6 +175,7 @@ def cmd_validate(args) -> int: X_val=X_val, y_val=y_val, require_full_reason_coverage=args.require_reasons, + require_selection_hygiene=args.require_selection_hygiene, ) print(json.dumps(report, indent=2)) return 0 if report["all_pass"] else 1 @@ -275,7 +276,12 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("--out", help="output file (default: stdout)") p.set_defaults(func=cmd_scorecard) - p = sub.add_parser("validate", help="run the 10-check validation framework") + p = sub.add_parser("validate", help="run the validation framework") + p.add_argument( + "--require-selection-hygiene", + action="store_true", + help="fail check 11 when the provenance block records a broken selection protocol", + ) p.add_argument("artifact") p.add_argument("--csv", help="validation CSV (features + outcome)") p.add_argument("--y-col", help="outcome column name in --csv") diff --git a/src/compileml/compile/distill.py b/src/compileml/compile/distill.py index 2e886d5..bca7a60 100644 --- a/src/compileml/compile/distill.py +++ b/src/compileml/compile/distill.py @@ -35,6 +35,7 @@ def train_whitebox( monotone_constraints=None, teacher_latent=None, sample_weight=None, + backend: str | None = None, ): """Fit a whitebox GBM to a target. @@ -54,6 +55,14 @@ def train_whitebox( the backend to ``HistGradientBoostingRegressor``; ``None`` (or all zeros) keeps the classic ``GradientBoostingRegressor``. + ``backend`` overrides that choice: ``"hist"`` for + ``HistGradientBoostingRegressor`` and ``"gbr"`` for the classic + ``GradientBoostingRegressor``. The histogram backend is the one to use + at scale — it bins features and runs multithreaded, and was measured at + 1–2 s where the classic backend took 18 s at 30k rows and 293 s at 300k + — but the default stays ``None`` (classic unless constrained) because + changing it would change every artifact built with the defaults. + ``sample_weight`` — one non-negative weight per row — tells the fit where fidelity matters. A whitebox has a fixed budget and, unweighted, spends it where most of the squared error is: the largest segment and the busiest @@ -97,7 +106,11 @@ def train_whitebox( raise ValueError("sample_weight must be finite, non-negative, and not all zero") feature_names = list(X.columns) if hasattr(X, "columns") else None cst = normalize_constraints(monotone_constraints, X_arr.shape[1], feature_names=feature_names) - if cst is not None: + if backend not in (None, "hist", "gbr"): + raise ValueError("backend must be None, 'hist' or 'gbr'") + if backend == "gbr" and cst is not None: + raise ValueError("monotone constraints need the 'hist' backend") + if cst is not None or backend == "hist": model = HistGradientBoostingRegressor( max_iter=n_estimators, max_depth=max_depth, diff --git a/src/compileml/compile/extract.py b/src/compileml/compile/extract.py index f0c937c..bcc8b66 100644 --- a/src/compileml/compile/extract.py +++ b/src/compileml/compile/extract.py @@ -369,7 +369,15 @@ def _extract_lightgbm(model) -> ExtractedModel: def extract_trees(model) -> ExtractedModel: - """Extract a fitted model into float tree arrays; dispatches on family.""" + """Extract a fitted model into float tree arrays; dispatches on family. + + An :class:`ExtractedModel` passes through unchanged. That is what lets a + caller extract a boosted model once and build artifacts from prefixes of + its trees — the first ``k`` trees of a boosted fit *are* the ``k``-tree + model — without refitting. + """ + if isinstance(model, ExtractedModel): + return model if _is_sklearn_hist(model): extracted = _extract_sklearn_hist(model) validate_extraction(extracted, model) diff --git a/src/compileml/select.py b/src/compileml/select.py new file mode 100644 index 0000000..5c4bbce --- /dev/null +++ b/src/compileml/select.py @@ -0,0 +1,978 @@ +"""Select the target and the configuration on data the report never sees. + +Three models with fixed jobs. The **ceiling** (the teacher) is the strongest +model the user can train under a declared budget; it prices compilation and +may supply soft targets, and it never ships. The **floor** — a WoE logistic +reference or a champion scorecard's Gini — says whether adopting the artifact +is worth it. The **candidate** is the depth-≤2 whitebox, one per +configuration, trained on labels, ceiling predictions, or a blend. + +Three disjoint partitions. **Fit** trains everything; **Select** chooses the +target and the configuration; **Report** is read once, for the number that +gets published. The selected configuration is refit on Fit ∪ Select before +Report is read, so the selection curve ranks configurations, not the artifact +that ships. + +Whether the whitebox should learn from the ceiling's probabilities or from +the labels is a parameter this module selects, not an assumption: on real +data, the whitebox trained on labels has beaten the distilled one. Every +step here is also a plain function, so the protocol can be followed by hand. + +Learning side: NumPy and scikit-learn throughout. Nothing here runs at +decision time. +""" + +from __future__ import annotations + +import hashlib +import warnings +from dataclasses import dataclass, field, replace + +import numpy as np +from sklearn.base import clone +from sklearn.metrics import roc_auc_score +from sklearn.model_selection import GroupShuffleSplit, StratifiedKFold, train_test_split + +from compileml.artifact import build_artifact +from compileml.bands import monotone_quantile_bands +from compileml.batch import score_batch +from compileml.compile import extract_trees, train_whitebox +from compileml.reference import fit_reference +from compileml.runtime.io import canonical_hash + +DEFAULT_ALPHAS = (0.0, 0.25, 0.5, 0.75, 1.0) + + +def default_grid() -> dict: + """The default search space: five targets, four tree counts, two depths. + + ``monotone`` holds ``None`` and/or ``"declared"``; the declared set comes + from ``compile_selected(monotone_constraints=...)`` and is never inferred. + Depth 3 is not in the grid because it forfeits exact attribution; add it + explicitly if that trade is acceptable. + """ + return { + "alpha": DEFAULT_ALPHAS, + "trees": (20, 40, 80, 160), + "depth": (1, 2), + "monotone": (None,), + } + + +# ------------------------------------------------------------------ metrics +def gini(y, score) -> float: + y = np.asarray(y, dtype=int) + if len(np.unique(y)) < 2: + return float("nan") + return 2.0 * float(roc_auc_score(y, np.asarray(score, dtype=float))) - 1.0 + + +def ks_statistic(y, score) -> float: + """Maximum separation between the score distributions of the two classes.""" + y = np.asarray(y, dtype=int) + score = np.asarray(score, dtype=float) + if y.sum() == 0 or y.sum() == len(y): + return float("nan") + order = np.argsort(score, kind="stable") + pos = np.cumsum(y[order]) / y.sum() + neg = np.cumsum(1 - y[order]) / (len(y) - y.sum()) + return float(np.max(np.abs(pos - neg))) + + +def brier(y, prob) -> float: + y = np.asarray(y, dtype=float) + return float(np.mean((np.asarray(prob, dtype=float) - y) ** 2)) + + +# --------------------------------------------------------------- partitions +@dataclass +class Partitions: + """Row indices of the three partitions, and how they were made.""" + + fit: np.ndarray + select: np.ndarray + report: np.ndarray + method: str + + @property + def dev(self) -> np.ndarray: + """Fit ∪ Select — what the selected configuration is refit on.""" + return np.concatenate([self.fit, self.select]) + + +def make_partitions( + y, + *, + method: str = "stratified", + fractions=(0.6, 0.2, 0.2), + date=None, + group=None, + explicit: dict | None = None, + seed: int = 42, +) -> Partitions: + """Split rows into Fit, Select and Report. + + - ``method="stratified"`` (default): stratified on the label. + - ``date``: Report is the latest slice by date; Fit and Select are split + within the earlier period. + - ``group``: split by group so one applicant lands in one partition. Not + stratified, which the method name records. + - ``explicit``: ``{"fit": idx, "select": idx, "report": idx}``, checked + for disjointness. + """ + y = np.asarray(y).reshape(-1) + n = len(y) + if explicit is not None: + parts = {k: np.asarray(explicit[k], dtype=int) for k in ("fit", "select", "report")} + for k, idx in parts.items(): + if idx.size == 0 or idx.min() < 0 or idx.max() >= n or len(np.unique(idx)) != len(idx): + raise ValueError(f"explicit {k} indices must be unique and within range") + for a, b in (("fit", "select"), ("fit", "report"), ("select", "report")): + if np.intersect1d(parts[a], parts[b]).size: + raise ValueError(f"explicit partitions overlap: {a} and {b}") + return Partitions(parts["fit"], parts["select"], parts["report"], "explicit") + + f_fit, f_select, f_report = (float(v) for v in fractions) + if min(f_fit, f_select, f_report) <= 0 or abs(f_fit + f_select + f_report - 1.0) > 1e-9: + raise ValueError("fractions must be positive and sum to 1") + idx = np.arange(n) + + if date is not None: + order = idx[np.argsort(np.asarray(date), kind="stable")] + n_report = max(1, int(round(n * f_report))) + report, rest = order[-n_report:], order[:-n_report] + method = "out_of_time" + else: + if group is not None: + splitter = GroupShuffleSplit(n_splits=1, test_size=f_report, random_state=seed) + rest_pos, report_pos = next(splitter.split(idx, groups=np.asarray(group))) + rest, report = idx[rest_pos], idx[report_pos] + method = "group" + else: + rest, report = train_test_split(idx, test_size=f_report, stratify=y, random_state=seed) + method = "stratified" + + select_share = f_select / (f_fit + f_select) + if group is not None: + splitter = GroupShuffleSplit(n_splits=1, test_size=select_share, random_state=seed + 1) + fit_pos, select_pos = next(splitter.split(rest, groups=np.asarray(group)[rest])) + fit, select = rest[fit_pos], rest[select_pos] + method = method if method != "stratified" else "group" + if date is not None: + method = "out_of_time_group" + else: + fit, select = train_test_split( + rest, test_size=select_share, stratify=y[rest], random_state=seed + 1 + ) + return Partitions(np.sort(fit), np.sort(select), np.sort(report), method) + + +def row_content_hashes(X, y) -> np.ndarray: + """A 128-bit digest per row of its features and label, for disjointness checks.""" + X = np.ascontiguousarray(np.asarray(X, dtype=np.float64)) + y = np.asarray(y).reshape(-1) + out = np.empty(len(y), dtype="S16") + for i in range(len(y)): + out[i] = hashlib.sha256(X[i].tobytes() + bytes([int(y[i])])).digest()[:16] + return out + + +def partition_hashes(hashes: np.ndarray, parts: Partitions) -> dict: + """One hex digest per partition, over its sorted row digests.""" + result = {} + for name in ("fit", "select", "report"): + rows = np.sort(hashes[getattr(parts, name)]) + result[name] = hashlib.sha256(b"".join(rows.tolist())).hexdigest() + return result + + +def duplicate_rows(hashes: np.ndarray, parts: Partitions) -> int: + """Rows in Select whose content appears in Fit, plus rows in Report whose + content appears in Fit ∪ Select. Legitimate in credit data (coarse bins, + repeat applicants), so a count rather than a failure.""" + fit_set = set(hashes[parts.fit].tolist()) + dup = sum(1 for h in hashes[parts.select].tolist() if h in fit_set) + dev_set = fit_set | set(hashes[parts.select].tolist()) + dup += sum(1 for h in hashes[parts.report].tolist() if h in dev_set) + return int(dup) + + +# ------------------------------------------------------------------ ceiling +def ceiling_predict(model, X) -> np.ndarray: + """A ceiling's score: positive-class probability, or the prediction.""" + if hasattr(model, "predict_proba"): + return np.asarray(model.predict_proba(X))[:, 1].astype(float) + return np.asarray(model.predict(X), dtype=float).reshape(-1) + + +def freeze_ceiling(model): + """The configuration to refit: a search's best estimator, or the model itself.""" + frozen = getattr(model, "best_estimator_", model) + try: + clone(frozen) + except Exception as exc: # pragma: no cover - depends on the user's object + raise TypeError( + "the ceiling factory must return a scikit-learn-compatible estimator " + "(or a fitted search exposing best_estimator_) so its frozen " + "configuration can be refit for out-of-fold predictions; otherwise pass " + "ceiling_oof= and ceiling_scores= instead" + ) from exc + return frozen + + +def params_hash(estimator) -> str: + """A digest of the frozen hyperparameters, so provenance can name what was refit.""" + try: + text = repr(sorted(estimator.get_params(deep=False).items())) + except Exception: # pragma: no cover + text = repr(estimator) + return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16] + + +@dataclass +class _Ceiling: + kind: str # "factory" | "predictions" + factory: object = None + frozen: object = None + oof: np.ndarray | None = None # full-length, for kind == "predictions" + scores: np.ndarray | None = None + budget: dict | None = None + factory_calls: int = 0 + fits: int = 0 + + def tune(self, X, y, idx): + """Call the factory once on ``idx`` and freeze what it returns.""" + self.frozen = freeze_ceiling(self.factory(X[idx], y[idx])) + self.factory_calls += 1 + self.fits += 1 + + def out_of_fold(self, X, y, idx, k: int, seed: int) -> np.ndarray: + """Cross-fitted predictions on ``idx``: k refits of the frozen configuration.""" + if self.kind == "predictions": + return self.oof[idx] + out = np.empty(len(idx), dtype=float) + folds = StratifiedKFold(n_splits=k, shuffle=True, random_state=seed) + for train_pos, held_pos in folds.split(idx, y[idx]): + model = clone(self.frozen).fit(X[idx[train_pos]], y[idx[train_pos]]) + self.fits += 1 + out[held_pos] = ceiling_predict(model, X[idx[held_pos]]) + return out + + def score(self, X, y, train_idx, eval_idx) -> np.ndarray: + """The frozen configuration refit on ``train_idx``, scored on ``eval_idx``.""" + if self.kind == "predictions": + return self.scores[eval_idx] + model = clone(self.frozen).fit(X[train_idx], y[train_idx]) + self.fits += 1 + return ceiling_predict(model, X[eval_idx]) + + @property + def family(self) -> str: + if self.kind == "predictions": + return "user_predictions" + cls = type(self.frozen) + return f"{cls.__module__}.{cls.__name__}" + + +# --------------------------------------------------------------------- grid +def _resolve_grid(grid, monotone_constraints, has_ceiling) -> tuple[dict, list[str]]: + g = dict(default_grid()) + g.update(grid or {}) + notes = [] + alphas = tuple(float(a) for a in g["alpha"]) + if not has_ceiling: + if any(a < 1.0 for a in alphas): + notes.append("no ceiling predictions: alpha forced to (1.0,) — labels only") + alphas = (1.0,) + else: + for endpoint in (0.0, 1.0): + if endpoint not in alphas: + alphas = tuple(sorted({*alphas, endpoint})) + notes.append(f"alpha endpoint {endpoint} added: endpoints are always searched") + if any(not 0.0 <= a <= 1.0 for a in alphas): + raise ValueError("alpha values must lie in [0, 1]") + trees = tuple(sorted({int(t) for t in g["trees"]})) + depths = tuple(sorted({int(d) for d in g["depth"]})) + if any(d > 2 for d in depths): + notes.append("depth > 2 in the grid: those candidates forfeit exact attribution") + monos = [] + for m in g["monotone"]: + if m is None: + monos.append(None) + elif m == "declared": + if monotone_constraints is None: + raise ValueError("grid asks for the declared monotone set but none was given") + monos.append(monotone_constraints) + else: + raise ValueError("grid['monotone'] entries must be None or 'declared'") + return {"alpha": alphas, "trees": trees, "depth": depths, "monotone": tuple(monos)}, notes + + +def _config_key(alpha, trees, depth, mono_index) -> tuple: + return (float(alpha), int(trees), int(depth), int(mono_index)) + + +def evaluate_grid( + X, + y, + train_idx, + eval_idx, + *, + grid: dict, + oof_train: np.ndarray | None, + feature_names, + n_bands: int, + backend: str, + learning_rate: float, + sample_weight=None, + seed: int = 42, +) -> dict[tuple, dict]: + """Every configuration in the grid, compiled on ``train_idx`` and scored on ``eval_idx``. + + One fit per (alpha, depth, monotone) at the largest tree count; the + smaller counts are prefixes of that fit, extracted once and truncated. + Each prefix is quantized, calibrated and banded on the training rows and + scored on the evaluation rows through the batch scorer — the compiled + artifact is what is ranked, not the float whitebox. + """ + X_tr, y_tr = X[train_idx], y[train_idx] + X_ev = X[eval_idx] + w_tr = None if sample_weight is None else np.asarray(sample_weight, dtype=float)[train_idx] + baseline = np.median(X_tr, axis=0) + positives = int(y_tr.sum()) + trees_wanted = set(grid["trees"]) + max_trees = max(grid["trees"]) + rows: dict[tuple, dict] = {} + + for alpha in grid["alpha"]: + target = y_tr.astype(float) if alpha >= 1.0 else alpha * y_tr + (1.0 - alpha) * oof_train + for depth in grid["depth"]: + for m_index, mono in enumerate(grid["monotone"]): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + model, _ = train_whitebox( + X_tr, + target, + n_estimators=max_trees, + max_depth=depth, + learning_rate=learning_rate, + random_state=seed, + monotone_constraints=mono, + sample_weight=w_tr, + backend=backend, + ) + extracted = extract_trees(model) + stages = {} + for i, pred in enumerate(model.staged_predict(X_tr), start=1): + if i in trees_wanted: + stages[i] = np.clip(pred, 0.0, 1.0) + for k in grid["trees"]: + prefix = replace(extracted, trees=extracted.trees[:k]) + latent = stages[k] + artifact = build_artifact( + prefix, + feature_names, + baseline, + monotone_quantile_bands(latent, y_tr, n_bands=n_bands), + calibration_latent=latent, + calibration_y=y_tr, + monotone_constraints=mono, + ) + scored = score_batch(artifact, X_ev) + rows[_config_key(alpha, k, depth, m_index)] = { + "alpha": float(alpha), + "n_estimators": int(k), + "max_depth": int(depth), + "monotone": mono is not None, + "eps": positives / (k * (2**depth - 1)), + "_raw": scored["raw_micro"], + "_pd": scored["pd"], + } + return rows + + +# ---------------------------------------------------------------- selection +def _metric(name: str, y, raw, pd) -> float: + if name == "gini": + return gini(y, raw) + if name == "brier": + return -brier(y, pd) # higher is better throughout selection + raise ValueError("selection_metric must be 'gini' or 'brier'") + + +def choose( + rows: dict[tuple, dict], y_eval, *, metric: str, tie_se: float, n_boot: int, seed: int +) -> dict: + """One-SE-simplest: the simplest configuration statistically tied with the best. + + Bootstrap the evaluation rows to get the standard error of the best + configuration's metric; everything within ``tie_se`` standard errors is + tied. Among the tied, prefer α = 1 (no training dependency on the + ceiling), then fewer trees, then lower depth. The α preference is a + governance prior, recorded as the tie rule, not an empirical claim. + """ + y_eval = np.asarray(y_eval, dtype=int) + for row in rows.values(): + row["gini"] = gini(y_eval, row["_raw"]) + row["brier"] = brier(y_eval, row["_pd"]) + row["score"] = _metric(metric, y_eval, row["_raw"], row["_pd"]) + best_key = max(rows, key=lambda k: rows[k]["score"]) + best = rows[best_key] + + rng = np.random.default_rng(seed) + n = len(y_eval) + samples = [] + for _ in range(n_boot): + pick = rng.integers(0, n, n) + samples.append(_metric(metric, y_eval[pick], best["_raw"][pick], best["_pd"][pick])) + se = float(np.std(samples, ddof=1)) if n_boot > 1 else 0.0 + + tied = [k for k, r in rows.items() if r["score"] >= best["score"] - tie_se * se] + chosen_key = min(tied, key=lambda k: (-k[0], k[1], k[2], k[3])) + for k, r in rows.items(): + r["tied"] = k in tied + r["selected"] = k == chosen_key + return { + "key": chosen_key, + "best_key": best_key, + "se": se, + "n_tied": len(tied), + "metric": metric, + "tie_se": tie_se, + "n_boot": n_boot, + } + + +# ------------------------------------------------------------------- result +@dataclass +class SelectionResult: + """What ``compile_selected`` returns: the artifact, the curve, and one report.""" + + artifact: dict | None + selection_curve: list[dict] + selected: dict + partitions: Partitions + floor_gate: dict + notes: list[str] + report_evaluations: int = 0 + _state: dict = field(default_factory=dict, repr=False) + + def report(self) -> dict: + """Evaluate the shipped artifact on Report, once. + + Gini, KS and Brier for artifact, ceiling and floor; retention against + the ceiling and the ratio against the floor at equal weight, each with + a bootstrap interval. The figures are written into the artifact's + provenance and the artifact is rehashed, so what ships carries its own + reported cost. A second call raises: nothing may be chosen after + Report is read, and this counter is hygiene rather than proof. + """ + if self.report_evaluations >= 1: + raise RuntimeError( + "Report has already been evaluated once; a second evaluation after " + "changing anything would invalidate the published figure. Rerun the " + "whole pipeline if the configuration changes." + ) + if self.artifact is None: + raise RuntimeError("no artifact: the selected configuration did not clear the floor") + s = self._state + y_rep = s["y"][self.partitions.report] + scored = score_batch(self.artifact, s["X"][self.partitions.report]) + art_raw, art_pd = scored["raw_micro"], scored["pd"] + ceil = s["ceiling_report"] + floor_scores, floor_gini = s["floor_report"] + + def block(raw, prob): + return { + "gini": gini(y_rep, raw), + "ks": ks_statistic(y_rep, raw), + "brier": brier(y_rep, prob) if prob is not None else None, + } + + art_m = block(art_raw, art_pd) + ceil_m = ( + block(ceil, ceil) if ceil is not None else {"gini": None, "ks": None, "brier": None} + ) + floor_m = ( + block(floor_scores, floor_scores) + if floor_scores is not None + else {"gini": floor_gini, "ks": None, "brier": None} + ) + + rng = np.random.default_rng(s["seed"] + 7) + n = len(y_rep) + ret, ratio, art_g = [], [], [] + for _ in range(s["n_boot"]): + pick = rng.integers(0, n, n) + g_art = gini(y_rep[pick], art_raw[pick]) + art_g.append(g_art) + if ceil is not None: + g_ceil = gini(y_rep[pick], ceil[pick]) + ret.append(100 * g_art / g_ceil if g_ceil > 0 else float("nan")) + if floor_scores is not None: + g_floor = gini(y_rep[pick], floor_scores[pick]) + ratio.append(100 * g_art / g_floor if g_floor > 0 else float("nan")) + elif floor_gini: + ratio.append(100 * g_art / floor_gini) + + def interval(values): + values = [v for v in values if v == v] + if not values: + return None + lo, hi = np.percentile(values, [2.5, 97.5]) + return [float(lo), float(hi)] + + report = { + "rows": int(n), + "artifact": art_m, + "ceiling": ceil_m, + "floor": floor_m, + "retention_pct": ( + 100 * art_m["gini"] / ceil_m["gini"] + if ceil_m["gini"] and ceil_m["gini"] > 0 + else None + ), + "retention_ci": interval(ret), + "floor_ratio_pct": ( + 100 * art_m["gini"] / floor_m["gini"] + if floor_m["gini"] and floor_m["gini"] > 0 + else None + ), + "floor_ratio_ci": interval(ratio), + "artifact_gini_ci": interval(art_g), + "n_boot": int(s["n_boot"]), + "evaluations": 1, + } + self.report_evaluations = 1 + prov = self.artifact["metadata"]["provenance"] + prov["report"] = _plain(report) + self.artifact["artifact_hash"] = canonical_hash(self.artifact) + report["artifact_hash"] = self.artifact["artifact_hash"] + return report + + +def _plain(value): + if isinstance(value, dict): + return {str(k): _plain(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_plain(v) for v in value] + if isinstance(value, np.generic): + return value.item() + return value + + +def _column(X_raw, spec, name): + if spec is None: + return None + if isinstance(spec, str): + if not hasattr(X_raw, "columns"): + raise ValueError(f"{name} is a column name but X has no columns") + return np.asarray(X_raw[spec]) + return np.asarray(spec).reshape(-1) + + +# ---------------------------------------------------------- the entry point +def compile_selected( + X, + y, + *, + ceiling=None, + ceiling_oof=None, + ceiling_scores=None, + ceiling_budget: dict | None = None, + reference="woe", + partitions="stratified", + date=None, + group=None, + row_id=None, + grid: dict | None = None, + monotone_constraints=None, + selection_metric: str = "gini", + tie_se: float = 1.0, + n_boot: int = 200, + k_folds: int = 5, + min_select_events: int = 300, + n_bands: int = 10, + feature_names=None, + reasons: dict | None = None, + sample_weight=None, + allow_below_floor: bool = False, + eps_warn: float = 10.0, + duplicate_threshold: float = 0.01, + backend: str = "hist", + learning_rate: float = 0.2, + seed: int = 42, + build_kwargs: dict | None = None, +) -> SelectionResult: + """Compile the whitebox whose target and configuration were chosen on Select. + + Args: + X, y: All rows and binary outcomes; the function partitions them. + ceiling: A callable ``(X_fit, y_fit) -> fitted model``. It is called + once, its configuration is frozen, and that configuration is + refit ``k_folds`` times for cross-fitted soft targets. One search + plus K fits — never K searches. + ceiling_oof, ceiling_scores: For users who can only supply + predictions: out-of-fold predictions used as soft targets, and + the ceiling's scores used for retention, both full-length. + Provenance then records ``cross_fitted: "user_asserted"``. + ceiling_budget: Free-form dict recorded in provenance — trials, CV + scheme, search space — so retention is comparable across artifacts. + reference: ``"woe"`` fits the WoE logistic floor inside Fit; a float + is a champion scorecard's Gini; ``None`` skips the floor gate, + with a warning. + partitions: ``"stratified"``, or an explicit dict of index arrays. + date, group, row_id: Arrays (or column names, when X has columns). + ``date`` makes Report the latest slice; ``group`` keeps one + applicant in one partition; ``row_id`` turns partition overlap + into a hard failure. + grid: See :func:`default_grid`. + selection_metric: ``"gini"`` (default) or ``"brier"``. + tie_se: Width of the tie band in standard errors. + min_select_events: Below this many events in Select, selection uses + nested cross-validation over Fit ∪ Select instead, with a warning. + allow_below_floor: Build the artifact even if the selected + configuration does not beat the floor on Select. + eps_warn: Provisional events-per-split threshold below which the + grid should hold interior α values; a diagnostic, never an input. + duplicate_threshold: Maximum share of Select and Report rows whose + content also appears in an earlier partition. + backend: ``"hist"`` (the fast histogram backend) or ``"gbr"``. + + Returns: + A :class:`SelectionResult`. Call ``.report()`` once for the figure. + """ + X_raw = X + X_arr = np.asarray(X, dtype=float) + if X_arr.ndim != 2: + raise ValueError("X must be a 2-D array of rows") + y_arr = np.asarray(y).reshape(-1) + if len(y_arr) != X_arr.shape[0]: + raise ValueError("X and y must have the same number of rows") + if not np.isin(y_arr, [0, 1]).all(): + raise ValueError("y must contain only 0 and 1") + y_arr = y_arr.astype(int) + names = ( + [str(n) for n in feature_names] + if feature_names is not None + else ( + [str(c) for c in X_raw.columns] + if hasattr(X_raw, "columns") + else [f"f{i}" for i in range(X_arr.shape[1])] + ) + ) + if len(names) != X_arr.shape[1]: + raise ValueError("feature_names length does not match X") + notes: list[str] = [] + + # ---------------------------------------------------------- partitions + date_arr, group_arr, row_ids = ( + _column(X_raw, v, n) for v, n in ((date, "date"), (group, "group"), (row_id, "row_id")) + ) + if isinstance(partitions, dict): + parts = make_partitions(y_arr, explicit=partitions, seed=seed) + elif partitions == "stratified": + parts = make_partitions(y_arr, date=date_arr, group=group_arr, seed=seed) + else: + raise ValueError("partitions must be 'stratified' or a dict of index arrays") + if row_ids is not None: + overlap = 0 + for a, b in ( + (parts.fit, parts.select), + (parts.fit, parts.report), + (parts.select, parts.report), + ): + overlap += int(np.intersect1d(row_ids[a], row_ids[b]).size) + if overlap: + raise ValueError(f"{overlap} row id(s) appear in more than one partition") + hashes = row_content_hashes(X_arr, y_arr) + duplicates = duplicate_rows(hashes, parts) + dup_fraction = duplicates / (len(parts.select) + len(parts.report)) + if dup_fraction > duplicate_threshold: + raise ValueError( + f"{duplicates} rows in Select or Report ({dup_fraction:.1%}) have content " + "identical to a " + f"row in an earlier partition, above duplicate_threshold={duplicate_threshold}. " + "Repeat applicants? Pass group= so each lands in one partition." + ) + + # ------------------------------------------------------------- ceiling + if ceiling is not None and (ceiling_oof is not None or ceiling_scores is not None): + raise ValueError( + "pass either ceiling= (a factory) or ceiling_oof=/ceiling_scores=, not both" + ) + if ceiling is not None: + ceil = _Ceiling(kind="factory", factory=ceiling, budget=ceiling_budget) + elif ceiling_oof is not None: + oof = np.asarray(ceiling_oof, dtype=float).reshape(-1) + scores = np.asarray( + ceiling_scores if ceiling_scores is not None else ceiling_oof, dtype=float + ).reshape(-1) + if len(oof) != len(y_arr) or len(scores) != len(y_arr): + raise ValueError("ceiling_oof and ceiling_scores must be full-length") + ceil = _Ceiling(kind="predictions", oof=oof, scores=scores, budget=ceiling_budget) + else: + ceil = None + notes.append("no ceiling: retention cannot be reported; the floor is the only yardstick") + + resolved, grid_notes = _resolve_grid(grid, monotone_constraints, ceil is not None) + notes.extend(grid_notes) + + # --------------------------------------------------------------- floor + if reference is None: + warnings.warn( + "no reference floor: the artifact cannot be compared with what a risk team " + "builds in an afternoon. Pass reference='woe' or a champion scorecard's Gini.", + stacklevel=2, + ) + floor_is_gini = isinstance(reference, (int, float)) and not isinstance(reference, bool) + + # ----------------------------------------------------------- selection + select_events = int(y_arr[parts.select].sum()) + nested = select_events < min_select_events + if nested: + warnings.warn( + f"Select holds {select_events} events, below min_select_events={min_select_events}: " + "selecting with nested cross-validation over Fit ∪ Select instead. Report stays " + "held out.", + stacklevel=2, + ) + sw = None if sample_weight is None else np.asarray(sample_weight, dtype=float).reshape(-1) + + def run_split(train_idx, eval_idx): + oof_train = None + if ceil is not None and any(a < 1.0 for a in resolved["alpha"]): + if ceil.kind == "factory" and (ceil.frozen is None or nested): + ceil.tune(X_arr, y_arr, train_idx) + oof_train = ceil.out_of_fold(X_arr, y_arr, train_idx, k_folds, seed) + elif ceil is not None and ceil.kind == "factory" and ceil.frozen is None: + ceil.tune(X_arr, y_arr, train_idx) + rows = evaluate_grid( + X_arr, + y_arr, + train_idx, + eval_idx, + grid=resolved, + oof_train=oof_train, + feature_names=names, + n_bands=n_bands, + backend=backend, + learning_rate=learning_rate, + sample_weight=sw, + seed=seed, + ) + floor_scores = None + if reference is not None and not floor_is_gini: + ref = fit_reference( + X_arr[train_idx], y_arr[train_idx], feature_names=names, random_state=seed + ) + floor_scores = ref.score(X_arr[eval_idx]) + return rows, floor_scores + + if not nested: + rows, floor_select = run_split(parts.fit, parts.select) + y_eval = y_arr[parts.select] + else: + dev = parts.dev + folds = StratifiedKFold(n_splits=k_folds, shuffle=True, random_state=seed) + pooled: dict[tuple, dict] = {} + floor_parts, y_parts = [], [] + for train_pos, held_pos in folds.split(dev, y_arr[dev]): + fold_rows, fold_floor = run_split(dev[train_pos], dev[held_pos]) + y_parts.append(y_arr[dev[held_pos]]) + if fold_floor is not None: + floor_parts.append(fold_floor) + for key, row in fold_rows.items(): + agg = pooled.setdefault( + key, + { + **{k: v for k, v in row.items() if not k.startswith("_")}, + "_raw": [], + "_pd": [], + }, + ) + agg["_raw"].append(row["_raw"]) + agg["_pd"].append(row["_pd"]) + rows = { + k: {**r, "_raw": np.concatenate(r["_raw"]), "_pd": np.concatenate(r["_pd"])} + for k, r in pooled.items() + } + y_eval = np.concatenate(y_parts) + floor_select = np.concatenate(floor_parts) if floor_parts else None + + selected = choose( + rows, y_eval, metric=selection_metric, tie_se=tie_se, n_boot=n_boot, seed=seed + ) + chosen = rows[selected["key"]] + eps_min = min(r["eps"] for r in rows.values()) + if eps_min < eps_warn and set(resolved["alpha"]) <= {0.0, 1.0} and ceil is not None: + warnings.warn( + f"minimum events per split in the grid is {eps_min:.1f}, below eps_warn={eps_warn}, " + "and the " + "alpha grid holds only endpoints; interior values (a blend of labels and ceiling " + "predictions) are worth searching at this event count. The threshold is provisional.", + stacklevel=2, + ) + + # ---------------------------------------------------------- floor gate + if reference is None: + floor_gini_select = None + elif floor_is_gini: + floor_gini_select = float(reference) + else: + floor_gini_select = gini(y_eval, floor_select) + clears = floor_gini_select is None or chosen["gini"] >= floor_gini_select + floor_gate = { + "passed": bool(clears), + "select_gini": chosen["gini"], + "floor_gini": floor_gini_select, + "enforced": bool(reference is not None and not allow_below_floor), + } + + curve = sorted( + ({k: v for k, v in r.items() if not k.startswith("_")} for r in rows.values()), + key=lambda r: (-r["score"], r["n_estimators"], r["max_depth"]), + ) + for r in curve: + r.pop("score", None) + + result_common = dict( + selection_curve=curve, + selected={ + "alpha": chosen["alpha"], + "n_estimators": chosen["n_estimators"], + "max_depth": chosen["max_depth"], + "monotone": chosen["monotone"], + "select_gini": chosen["gini"], + "select_brier": chosen["brier"], + "eps": chosen["eps"], + **{k: v for k, v in selected.items() if k not in ("key", "best_key")}, + "nested": nested, + }, + partitions=parts, + floor_gate=floor_gate, + notes=notes, + ) + if not clears and not allow_below_floor: + return SelectionResult(artifact=None, **result_common) + + # --------------------------------------------------------- final refit + dev = parts.dev + alpha = chosen["alpha"] + mono = monotone_constraints if chosen["monotone"] else None + cross_fitted: object = None + if alpha < 1.0: + if ceil.kind == "factory" and (ceil.frozen is None or nested): + ceil.tune(X_arr, y_arr, dev) + oof_dev = ceil.out_of_fold(X_arr, y_arr, dev, k_folds, seed) + target = alpha * y_arr[dev] + (1.0 - alpha) * oof_dev + cross_fitted = True if ceil.kind == "factory" else "user_asserted" + else: + target = y_arr[dev].astype(float) + if ceil is not None and ceil.kind == "factory" and (ceil.frozen is None or nested): + ceil.tune(X_arr, y_arr, dev) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + model, _ = train_whitebox( + X_arr[dev], + target, + n_estimators=chosen["n_estimators"], + max_depth=chosen["max_depth"], + learning_rate=learning_rate, + random_state=seed, + monotone_constraints=mono, + sample_weight=None if sw is None else sw[dev], + backend=backend, + ) + latent = np.clip(model.predict(X_arr[dev]), 0.0, 1.0) + part_hashes = partition_hashes(hashes, parts) + provenance = { + "target": {"alpha": alpha, "uses_ceiling": alpha < 1.0, "loss": "squared_error"}, + "selection": { + "metric": selection_metric, + "rule": "one_se_simplest", + "tie_se": tie_se, + "se": selected["se"], + "n_tied": selected["n_tied"], + "n_configs": len(rows), + "n_boot": n_boot, + "nested": nested, + "k_folds": k_folds if nested else None, + "tie_rule": "prefer alpha=1, then fewer trees, then lower depth — a governance prior", + }, + "partitions": { + "method": parts.method + ("_nested" if nested else ""), + "fit_rows": int(len(parts.fit)), + "select_rows": int(len(parts.select)), + "report_rows": int(len(parts.report)), + "content_hashes": part_hashes, + "duplicate_rows": duplicates, + "duplicate_fraction": dup_fraction, + "duplicate_threshold": duplicate_threshold, + "row_id_overlap": 0 if row_ids is not None else None, + }, + "soft_targets": {"cross_fitted": cross_fitted, "k": k_folds if alpha < 1.0 else None}, + "ceiling": ( + None + if ceil is None + else { + "kind": ceil.kind, + "family": ceil.family, + "params_hash": params_hash(ceil.frozen) if ceil.kind == "factory" else None, + "budget": ceil.budget, + "factory_calls": ceil.factory_calls, + "fits": ceil.fits, + } + ), + "floor": ( + None + if reference is None + else { + "kind": "gini" if floor_is_gini else "woe_logit", + "select_gini": floor_gini_select, + } + ), + "eps_min": eps_min, + "eps_warn": eps_warn, + "report": None, + } + build_kwargs = dict(build_kwargs or {}) + metadata = {**build_kwargs.pop("metadata", {}), "provenance": _plain(provenance)} + artifact = build_artifact( + model, + names, + np.median(X_arr[dev], axis=0), + monotone_quantile_bands(latent, y_arr[dev], n_bands=n_bands), + calibration_latent=latent, + calibration_y=y_arr[dev], + monotone_constraints=mono, + reasons=reasons, + metadata=metadata, + X_sample=X_arr[dev][:500], + **build_kwargs, + ) + + ceiling_report = ceil.score(X_arr, y_arr, dev, parts.report) if ceil is not None else None + if reference is None: + floor_report = (None, None) + elif floor_is_gini: + floor_report = (None, float(reference)) + else: + ref = fit_reference(X_arr[dev], y_arr[dev], feature_names=names, random_state=seed) + floor_report = (ref.score(X_arr[parts.report]), None) + if ceil is not None: + artifact["metadata"]["provenance"]["ceiling"]["fits"] = ceil.fits + artifact["artifact_hash"] = canonical_hash(artifact) + + return SelectionResult( + artifact=artifact, + _state={ + "X": X_arr, + "y": y_arr, + "ceiling_report": ceiling_report, + "floor_report": floor_report, + "n_boot": n_boot, + "seed": seed, + }, + **result_common, + ) diff --git a/src/compileml/tune/sweeps.py b/src/compileml/tune/sweeps.py index caecb08..1fcb72c 100644 --- a/src/compileml/tune/sweeps.py +++ b/src/compileml/tune/sweeps.py @@ -48,7 +48,7 @@ def sweep_whitebox( *, trees_grid=(10, 20, 40, 80, 160), depth_grid=(1, 2, 3), - alpha_grid=(0.0,), + alpha_grid=None, learning_rate: float = 0.2, random_state: int = 42, X_val=None, @@ -102,6 +102,24 @@ def sweep_whitebox( weighted sweep beside an unweighted one shows what moving the budget buys one segment and costs the others. """ + if alpha_grid is None: + warnings.warn( + "sweep_whitebox defaulted to pure distillation, alpha_grid=(0.0,), which real " + "data has falsified: a whitebox trained on labels has beaten the distilled one. " + "Pass alpha_grid explicitly (at least the endpoints (0.0, 1.0)), or use " + "compile_selected, which selects alpha on data it does not report on. The " + "default becomes (0.0, 0.5, 1.0) in 1.0.", + FutureWarning, + stacklevel=2, + ) + alpha_grid = (0.0,) + if X_val is not None: + warnings.warn( + "one holdout used both to choose a configuration and to report its retention " + "leaks: the figure chosen on it is optimistic. Keep it for choosing, and report " + "on rows never used to choose. compile_selected keeps Select and Report apart.", + stacklevel=2, + ) X_arr = np.asarray(X, dtype=float) y_arr = np.asarray(y, dtype=int).reshape(-1) has_teacher = teacher_latent is not None diff --git a/src/compileml/validate/framework.py b/src/compileml/validate/framework.py index 280a84a..94a9c83 100644 --- a/src/compileml/validate/framework.py +++ b/src/compileml/validate/framework.py @@ -25,6 +25,9 @@ 9. monotone_constraints declared directions re-verified against the shipped integer trees (skipped when the artifact declares none) + 11. selection_hygiene the provenance block, when present, records a + protocol that was followed: cross-fitted soft + targets, disjoint partitions, one Report read 10. reference_floor the artifact out-scores a reference model on the same data — the floor that teacher retention alone can never supply @@ -75,6 +78,7 @@ def validate_artifact( require_full_reason_coverage: bool = False, reference=None, require_reference_floor: bool = False, + require_selection_hygiene: bool = False, seed: int = 42, ) -> dict: """Run the ten-check framework. Checks lacking inputs skip, not fail. @@ -97,6 +101,12 @@ def validate_artifact( require_reference_floor: Make check 10 *fail* when the artifact does not clear the reference, rather than recording it as evidence (the ``max_within_band_auc`` precedent). + require_selection_hygiene: Make check 11 *fail* on a provenance + block that records a broken protocol — soft targets that were not + cross-fitted, partitions that overlap, Report read more than once + — rather than listing the issues as evidence. Check 11 skips when + the artifact carries no provenance block; it can verify what the + block records, not that the block is honest. Returns: {"all_pass": bool, "checks": {name: {"pass", "skipped", ...evidence}}} @@ -371,4 +381,65 @@ def bootstrap_cutoffs(): else: checks["10_reference_floor"] = {"pass": True, "skipped": True} + # ------------------------------------------------ 11 selection hygiene + provenance = (artifact.get("metadata") or {}).get("provenance") + if provenance: + checks["11_selection_hygiene"] = _selection_hygiene(provenance, require_selection_hygiene) + else: + checks["11_selection_hygiene"] = {"pass": True, "skipped": True} + return {"all_pass": all(c["pass"] for c in checks.values()), "checks": checks} + + +def _selection_hygiene(provenance: dict, required: bool) -> dict: + """Check 11: does the provenance block record a protocol that was followed? + + Issues fail the check when it is required; notes never do. The check reads + what ``compile_selected`` wrote and cannot verify it against data it does + not have, which is why it is advisory by default. + """ + issues: list[str] = [] + notes: list[str] = [] + target = provenance.get("target") or {} + alpha = float(target.get("alpha", 1.0)) + cross_fitted = (provenance.get("soft_targets") or {}).get("cross_fitted") + if alpha < 1.0: + if cross_fitted is True: + pass + elif cross_fitted == "user_asserted": + notes.append("soft-target cross-fitting is asserted by the user, not verified") + else: + issues.append( + "soft targets were not cross-fitted (alpha < 1 with in-sample ceiling predictions)" + ) + + parts = provenance.get("partitions") or {} + hashes = parts.get("content_hashes") or {} + if hashes and len(set(hashes.values())) != len(hashes): + issues.append("partition content hashes are not distinct") + if parts.get("row_id_overlap"): + issues.append(f"{parts['row_id_overlap']} row id(s) appear in more than one partition") + dup, threshold = parts.get("duplicate_fraction"), parts.get("duplicate_threshold") + if dup is not None and threshold is not None and dup > threshold: + issues.append(f"duplicate fraction {dup:.4f} exceeds threshold {threshold}") + + report = provenance.get("report") + evaluations = report.get("evaluations") if report else None + if report is None: + issues.append("Report has not been evaluated: the artifact carries no reported figure") + elif evaluations != 1: + issues.append(f"Report was evaluated {evaluations} times; the protocol allows one") + elif parts.get("report_rows") is not None and report.get("rows") != parts.get("report_rows"): + issues.append("the reported row count does not match the Report partition") + + return { + "pass": bool(not issues or not required), + "skipped": False, + "required": bool(required), + "alpha": alpha, + "cross_fitted": cross_fitted, + "report_evaluations": evaluations, + "duplicate_fraction": dup, + "issues": issues, + "notes": notes, + } diff --git a/tests/test_batch.py b/tests/test_batch.py new file mode 100644 index 0000000..cfe63c8 --- /dev/null +++ b/tests/test_batch.py @@ -0,0 +1,64 @@ +"""The batch scorer is the runtime's score path, vectorised: same integers, every row.""" + +import copy + +import numpy as np +import pytest + +from compileml.batch import score_batch +from compileml.runtime import ArtifactError, decide, load_artifact + +FIELDS = ("raw_micro", "latent_micro", "latent_int", "band_idx", "pd_ppm") + + +@pytest.fixture(scope="module") +def artifact(): + return load_artifact("tests/data/reference_artifact.json") + + +@pytest.fixture(scope="module") +def rows(artifact): + """Ordinary rows, missing values, and values far enough out to hit both clamps.""" + p = len(artifact["features"]["names"]) + rng = np.random.default_rng(0) + X = rng.standard_normal((2000, p)) * 2 + X[::7, 0] = np.nan + X[::11, 3] = 1e6 + X[::13, 4] = -1e6 + return X + + +def _agrees(art, X): + batch = score_batch(art, X) + for i, row in enumerate(X): + d = decide(art, [None if v != v else float(v) for v in row], explain=False) + assert tuple(d[f] for f in FIELDS) == tuple(int(batch[f][i]) for f in FIELDS), i + + +def test_bit_identical_to_decide(artifact, rows): + _agrees(artifact, rows) + + +@pytest.mark.parametrize("variant", ["step", "none", "float32"]) +def test_bit_identical_across_calibration_modes_and_precision(artifact, rows, variant): + art = copy.deepcopy(artifact) + if variant == "step": + art["calibration"]["mode"] = "step" + elif variant == "none": + art["calibration"] = None + else: + art["model"]["input_precision"] = "float32" + _agrees(art, rows) + + +def test_reject_policy_refuses_missing_values(artifact, rows): + art = copy.deepcopy(artifact) + art["features"]["missing_policy"] = "reject" + with pytest.raises(ArtifactError, match="reject"): + score_batch(art, rows) + score_batch(art, np.nan_to_num(rows)) # no missing values: fine + + +def test_wrong_width_is_refused(artifact, rows): + with pytest.raises(ArtifactError, match="feature columns"): + score_batch(artifact, rows[:, :-1]) diff --git a/tests/test_select.py b/tests/test_select.py new file mode 100644 index 0000000..68ad2c6 --- /dev/null +++ b/tests/test_select.py @@ -0,0 +1,447 @@ +"""compile_selected: the protocol, and the properties #75 promises. + +The fixture data is small and the grids are short so the suite stays fast; +the properties do not depend on scale. +""" + +import copy +import warnings + +import numpy as np +import pytest +from sklearn.base import BaseEstimator, ClassifierMixin +from sklearn.ensemble import HistGradientBoostingClassifier + +import compileml +from compileml.artifact import build_artifact +from compileml.bands import monotone_quantile_bands +from compileml.batch import score_batch +from compileml.compile import extract_trees, train_whitebox +from compileml.runtime import canonical_hash, verify_artifact +from compileml.select import ( + choose, + compile_selected, + duplicate_rows, + make_partitions, + row_content_hashes, +) +from compileml.tune import sweep_whitebox +from compileml.validate import validate_artifact + +N, P = 4000, 6 +FEATURES = [f"f{i}" for i in range(P)] +SMALL_GRID = {"alpha": (0.0, 1.0), "trees": (20, 40), "depth": (1, 2)} + + +class CountingCeiling(ClassifierMixin, BaseEstimator): + """A ceiling that counts its fits, so the protocol's fit budget is testable.""" + + fits = 0 + + def __init__(self, max_iter=60): + self.max_iter = max_iter + + def fit(self, X, y): + CountingCeiling.fits += 1 + self.model_ = HistGradientBoostingClassifier( + max_iter=self.max_iter, max_depth=3, learning_rate=0.1, random_state=0 + ).fit(X, y) + self.classes_ = self.model_.classes_ + return self + + def predict_proba(self, X): + return self.model_.predict_proba(X) + + def predict(self, X): + return self.model_.predict(X) + + +@pytest.fixture(scope="module") +def data(): + rng = np.random.default_rng(1) + X = rng.standard_normal((N, P)) + logit = 1.2 * X[:, 0] - 0.8 * X[:, 1] + 0.7 * X[:, 2] * X[:, 3] - 1.0 + y = (rng.random(N) < 1 / (1 + np.exp(-logit))).astype(int) + return X, y + + +@pytest.fixture(scope="module") +def run(data): + """One flat run with a counting factory, shared by the tests that read it.""" + X, y = data + calls = {"factory": 0} + CountingCeiling.fits = 0 + + def factory(Xf, yf): + calls["factory"] += 1 + return CountingCeiling().fit(Xf, yf) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + result = compile_selected( + X, + y, + ceiling=factory, + reference="woe", + grid=SMALL_GRID, + n_boot=30, + k_folds=3, + min_select_events=50, + seed=3, + ) + return result, calls + + +# ---------------------------------------------------------------- partitions +def test_partitions_are_disjoint_and_cover_every_row(data): + _, y = data + parts = make_partitions(y, seed=0) + all_idx = np.sort(np.concatenate([parts.fit, parts.select, parts.report])) + assert np.array_equal(all_idx, np.arange(N)) + assert abs(len(parts.report) / N - 0.2) < 0.01 and abs(len(parts.fit) / N - 0.6) < 0.01 + # stratified: event rates agree + rates = [y[getattr(parts, k)].mean() for k in ("fit", "select", "report")] + assert max(rates) - min(rates) < 0.02 + + +def test_out_of_time_report_is_the_latest_slice(data): + _, y = data + date = np.arange(N) # row i happened on day i + parts = make_partitions(y, date=date, seed=0) + assert parts.report.min() > max(parts.fit.max(), parts.select.max()) + assert parts.method == "out_of_time" + + +def test_group_split_keeps_each_group_in_one_partition(data): + _, y = data + group = np.arange(N) // 10 # ten rows per applicant + parts = make_partitions(y, group=group, seed=0) + seen = {} + for name in ("fit", "select", "report"): + for g in np.unique(group[getattr(parts, name)]): + assert seen.setdefault(g, name) == name, f"group {g} straddles partitions" + assert parts.method == "group" + + +def test_explicit_partitions_must_not_overlap(data): + _, y = data + with pytest.raises(ValueError, match="overlap"): + make_partitions(y, explicit={"fit": [0, 1, 2], "select": [2, 3], "report": [4]}) + with pytest.raises(ValueError, match="sum to 1"): + make_partitions(y, fractions=(0.5, 0.3, 0.3)) + + +def test_content_duplicates_are_counted_and_thresholded(data): + X, y = data + hashes = row_content_hashes(X, y) + parts = make_partitions(y, seed=0) + assert duplicate_rows(hashes, parts) == 0 + X_dup = X.copy() + X_dup[parts.report[:100]] = X[parts.fit[:100]] # report rows copied from fit + y_dup = y.copy() + y_dup[parts.report[:100]] = y[parts.fit[:100]] + assert duplicate_rows(row_content_hashes(X_dup, y_dup), parts) == 100 + with pytest.raises(ValueError, match="identical"): + compile_selected( + X_dup, + y_dup, + partitions={"fit": parts.fit, "select": parts.select, "report": parts.report}, + reference=0.1, + grid={"alpha": (1.0,), "trees": (20,), "depth": (1,)}, + n_boot=5, + min_select_events=10, + duplicate_threshold=0.01, + ) + + +def test_row_id_overlap_is_a_hard_failure(data): + X, y = data + parts = make_partitions(y, seed=0) + row_id = np.arange(N) + row_id[parts.report[0]] = row_id[parts.fit[0]] # one id in two partitions + with pytest.raises(ValueError, match="row id"): + compile_selected( + X, + y, + row_id=row_id, + partitions={"fit": parts.fit, "select": parts.select, "report": parts.report}, + reference=0.1, + grid={"alpha": (1.0,), "trees": (20,), "depth": (1,)}, + n_boot=5, + ) + + +# ------------------------------------------------------------ the prefix trick +def test_prefix_of_a_fit_is_the_smaller_fit(data): + """The first 20 trees of an 80-tree fit compile to the same artifact as a 20-tree fit.""" + X, y = data + target = y.astype(float) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + big, _ = train_whitebox(X, target, n_estimators=80, backend="hist", random_state=0) + small, _ = train_whitebox(X, target, n_estimators=20, backend="hist", random_state=0) + extracted = extract_trees(big) + prefix = copy.copy(extracted) + prefix.trees = extracted.trees[:20] + latent = np.clip(small.predict(X), 0, 1) + bands = monotone_quantile_bands(latent, y, n_bands=6) + art_prefix = build_artifact(prefix, FEATURES, np.median(X, axis=0), bands) + art_small = build_artifact(small, FEATURES, np.median(X, axis=0), bands) + assert np.array_equal( + score_batch(art_prefix, X)["raw_micro"], score_batch(art_small, X)["raw_micro"] + ) + + +# ------------------------------------------------------------------ ceiling +def test_ceiling_is_tuned_once_and_refit_frozen(run): + """One search, K fits for out-of-fold targets, one refit for Report scoring.""" + result, calls = run + k = 3 + assert calls["factory"] == 1 + prov = result.artifact["metadata"]["provenance"]["ceiling"] + assert prov["factory_calls"] == 1 + expected = 1 + k + 1 # tune, out-of-fold on Fit, refit on Fit ∪ Select for Report + if result.selected["alpha"] < 1.0: + expected += k # out-of-fold regenerated on Fit ∪ Select for the final refit + assert prov["fits"] == expected == CountingCeiling.fits + + +# ---------------------------------------------------------------- selection +def _rows(y, preds): + return { + key: { + "alpha": key[0], + "n_estimators": key[1], + "max_depth": key[2], + "monotone": False, + "eps": 50.0, + "_raw": raw, + "_pd": np.clip(raw / raw.max(), 0, 1), + } + for key, raw in preds.items() + } + + +def test_tie_rule_prefers_alpha_one_then_fewer_trees_then_lower_depth(): + rng = np.random.default_rng(0) + y = (rng.random(500) < 0.3).astype(int) + same = (y * 1000 + rng.integers(0, 200, 500)).astype(np.int64) + rows = _rows( + y, + { + (0.0, 20, 1, 0): same, + (1.0, 80, 2, 0): same, + (1.0, 20, 2, 0): same, + (1.0, 20, 1, 0): same, + }, + ) + chosen = choose(rows, y, metric="gini", tie_se=1.0, n_boot=20, seed=0) + assert chosen["key"] == (1.0, 20, 1, 0) + assert chosen["n_tied"] == 4 + + +def test_a_clearly_better_configuration_wins_regardless_of_the_prior(): + rng = np.random.default_rng(0) + y = (rng.random(500) < 0.3).astype(int) + sharp = (y * 1000 + rng.integers(0, 100, 500)).astype(np.int64) + noise = rng.integers(0, 1000, 500).astype(np.int64) + rows = _rows(y, {(0.0, 80, 2, 0): sharp, (1.0, 20, 1, 0): noise}) + chosen = choose(rows, y, metric="gini", tie_se=1.0, n_boot=20, seed=0) + assert chosen["key"] == (0.0, 80, 2, 0) + assert chosen["n_tied"] == 1 + + +def test_selection_curve_and_selected_agree(run): + result, _ = run + curve = result.selection_curve + assert len(curve) == 2 * 2 * 2 # alpha x trees x depth + assert sum(r["selected"] for r in curve) == 1 + top = next(r for r in curve if r["selected"]) + assert (top["alpha"], top["n_estimators"], top["max_depth"]) == ( + result.selected["alpha"], + result.selected["n_estimators"], + result.selected["max_depth"], + ) + assert all("_raw" not in r for r in curve) + assert all(r["eps"] > 0 for r in curve) + + +# ------------------------------------------------------ floor and no ceiling +def test_floor_gate_stops_without_an_artifact(data): + X, y = data + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + stopped = compile_selected( + X[:1500], + y[:1500], + reference=0.99, + grid={"alpha": (1.0,), "trees": (20,), "depth": (2,)}, + n_boot=5, + min_select_events=10, + seed=3, + ) + forced = compile_selected( + X[:1500], + y[:1500], + reference=0.99, + grid={"alpha": (1.0,), "trees": (20,), "depth": (2,)}, + n_boot=5, + min_select_events=10, + seed=3, + allow_below_floor=True, + ) + assert stopped.artifact is None and stopped.floor_gate["passed"] is False + with pytest.raises(RuntimeError, match="floor"): + stopped.report() + assert forced.artifact is not None + + +def test_no_ceiling_forces_labels_and_reports_no_retention(data): + X, y = data + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + result = compile_selected( + X[:1500], + y[:1500], + reference=0.2, + grid={"trees": (20,), "depth": (2,)}, + n_boot=5, + min_select_events=10, + seed=3, + ) + assert result.selected["alpha"] == 1.0 + assert any("forced" in n for n in result.notes) + report = result.report() + assert report["retention_pct"] is None and report["floor_ratio_pct"] > 0 + + +# --------------------------------------------------------- report and hash +def test_report_is_read_once_and_written_into_the_hash(run): + result, _ = run + before = result.artifact["artifact_hash"] + assert verify_artifact(result.artifact) + report = result.report() + prov = result.artifact["metadata"]["provenance"] + assert prov["report"]["evaluations"] == 1 + assert report["artifact_hash"] == result.artifact["artifact_hash"] != before + assert verify_artifact(result.artifact) + assert report["retention_ci"][0] <= report["retention_pct"] <= report["retention_ci"][1] + assert report["floor_ratio_ci"][0] <= report["floor_ratio_pct"] <= report["floor_ratio_ci"][1] + with pytest.raises(RuntimeError, match="already been evaluated"): + result.report() + + +def test_selection_never_computes_attribution(data, monkeypatch): + import compileml.runtime.explain as explain + + def boom(*args, **kwargs): + raise AssertionError("attribution was computed during selection") + + monkeypatch.setattr(explain, "contributions_half_micro", boom) + X, y = data + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + result = compile_selected( + X[:1500], + y[:1500], + reference=0.1, + grid={"alpha": (1.0,), "trees": (20,), "depth": (1,)}, + n_boot=5, + min_select_events=10, + seed=3, + ) + assert result.artifact is not None + + +# ------------------------------------------------------------------ nested +def test_small_select_falls_back_to_nested_cv(data): + X, y = data + with pytest.warns(UserWarning, match="nested"): + result = compile_selected( + X[:1200], + y[:1200], + reference=0.1, + grid={"alpha": (1.0,), "trees": (20,), "depth": (1,)}, + n_boot=5, + k_folds=3, + min_select_events=10_000, + seed=3, + ) + assert result.selected["nested"] is True + assert result.artifact["metadata"]["provenance"]["partitions"]["method"].endswith("_nested") + assert result.report()["rows"] == len(result.partitions.report) + + +# ---------------------------------------------------------------- check 11 +def test_check_11_passes_on_a_reported_artifact_and_catches_tampering(run, data): + result, _ = run + X, y = data + if result.report_evaluations == 0: + result.report() + idx = result.partitions.report + v = validate_artifact(result.artifact, X[idx], y[idx], require_selection_hygiene=True) + c11 = v["checks"]["11_selection_hygiene"] + assert c11["pass"] and not c11["skipped"] and c11["issues"] == [] + + tampered = copy.deepcopy(result.artifact) + prov = tampered["metadata"]["provenance"] + prov["report"]["evaluations"] = 2 + prov["target"]["alpha"] = 0.5 + prov["soft_targets"]["cross_fitted"] = False + tampered["artifact_hash"] = canonical_hash(tampered) + advisory = validate_artifact(tampered, X[idx], y[idx])["checks"]["11_selection_hygiene"] + assert advisory["pass"] and len(advisory["issues"]) == 2 + required = validate_artifact(tampered, X[idx], y[idx], require_selection_hygiene=True) + assert required["checks"]["11_selection_hygiene"]["pass"] is False + assert required["all_pass"] is False + + plain = copy.deepcopy(result.artifact) + del plain["metadata"]["provenance"] + plain["artifact_hash"] = canonical_hash(plain) + assert validate_artifact(plain)["checks"]["11_selection_hygiene"]["skipped"] + + +# --------------------------------------------------------- warnings and API +def test_sweep_whitebox_warns_about_its_old_defaults(data): + X, y = data + teacher = 1 / (1 + np.exp(-(X[:, 0] - 1))) + with pytest.warns(FutureWarning, match="alpha_grid"): + sweep_whitebox( + X[:800], teacher[:800], y[:800], trees_grid=(10,), depth_grid=(1,), reference=0.1 + ) + with pytest.warns(UserWarning, match="choose"): + sweep_whitebox( + X[:800], + teacher[:800], + y[:800], + trees_grid=(10,), + depth_grid=(1,), + alpha_grid=(0.0,), + X_val=X[800:1000], + y_val=y[800:1000], + teacher_latent_val=teacher[800:1000], + reference=0.1, + ) + + +def test_eps_warning_when_endpoints_only_and_events_are_scarce(data): + X, y = data + with pytest.warns(UserWarning, match="events per split"): + compile_selected( + X[:600], + y[:600], + ceiling=lambda Xf, yf: CountingCeiling(max_iter=20).fit(Xf, yf), + reference=0.1, + grid={"alpha": (0.0, 1.0), "trees": (160,), "depth": (2,)}, + n_boot=5, + k_folds=2, + min_select_events=1, + seed=3, + ) + + +def test_compile_selected_is_importable_from_the_root_lazily(): + assert compileml.compile_selected is compile_selected + missing = "no_such_thing" + with pytest.raises(AttributeError): + getattr(compileml, missing) diff --git a/tests/test_validate.py b/tests/test_validate.py index a07e423..1890ec8 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -68,7 +68,9 @@ def test_all_checks_pass(fitted, reference): reference=reference, # check 10 skips without one ) assert report["all_pass"], {k: v for k, v in report["checks"].items() if not v["pass"]} - assert not any(c["skipped"] for c in report["checks"].values()) + # Check 11 reads a provenance block that only compile_selected writes. + skipped = [k for k, c in report["checks"].items() if c["skipped"]] + assert skipped == ["11_selection_hygiene"] assert report["checks"]["8_reason_coverage"]["coverage"] == 1.0 assert report["checks"]["9_monotone_constraints"]["constrained_features"] == ["x0", "x1", "x2"] From 70205f692cbe6bfe8ef4b3a03df7bcd6e6d5d7bd Mon Sep 17 00:00:00 2001 From: Carlos Ortiz Date: Sun, 27 Sep 2026 10:22:14 -0600 Subject: [PATCH 2/5] Run the benchmark through compile_selected and stop describing distillation as the method The benchmark selects the whitebox on Select from the default grid of 40 configurations, refits the winner on Fit union Select, and reads retention once from Report. It chose alpha = 0.75, 80 trees, depth 2: a blend of labels and the ceiling's cross-fitted predictions, with both pure targets scoring lower on Select. Retention is 98.04% on Report with a 95% interval of 97.38 to 98.72, against 97.9% before on a holdout that had also chosen the configuration, and the artifact stands at 103.41% of a WoE logistic floor. The ceiling is the histogram GBM, fixed and recorded as unsearched, so the whole run takes minutes; the determinism check reruns the entire protocol and requires the same configuration and the same hash. results.json gains the selected configuration, intervals, floor figures, KS, Brier and events per split, and drops distill_spearman_train; the selection curve is committed beside it. The documentation stops describing distillation as what CompileML does. "Compile" is the verb; the strongest model is the ceiling (the teacher), a measuring instrument that never ships; retention is defined at first use as artifact Gini over ceiling Gini on Report, against a ceiling tuned under a declared budget; the floor is reported at equal weight. The README quick example runs through compile_selected, the quickstart keeps the by-hand path, the tuning guide gains "Let the data choose", the FAQ answers "the pipeline selects it", the validation pages count eleven checks, and the API reference lists the selection and batch-scoring functions. No API name changes. Closes #75 --- CHANGELOG.md | 10 + CONTRIBUTING.md | 2 +- README.md | 100 ++++--- benchmarks/results.json | 92 ++++--- benchmarks/run_benchmarks.py | 167 ++++++++---- benchmarks/selection_curve.json | 445 ++++++++++++++++++++++++++++++++ docs/ARTIFACT_SPEC.md | 2 +- docs/concepts/attribution.md | 10 +- docs/concepts/bands.md | 2 +- docs/concepts/where-it-fits.md | 2 +- docs/faq.md | 35 +-- docs/howto/tuning.md | 133 ++++++++-- docs/howto/validate.md | 3 +- docs/index.md | 1 + docs/quickstart.md | 26 +- docs/reference/api.md | 16 ++ docs/reference/cli.md | 3 +- 17 files changed, 863 insertions(+), 186 deletions(-) create mode 100644 benchmarks/selection_curve.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 7207a1d..847fa34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,16 @@ inside each artifact (`schema_version`). - `sweep_whitebox` warns when `alpha_grid` is not passed — its default of pure distillation is the assumption real data falsified — and when `X_val` is used both to choose and to report. The defaults switch in 1.0. +- The documentation stops describing distillation as what CompileML does. + "Compile" is the verb; the strongest model is the *ceiling* (the teacher), + a measuring instrument that never ships; *retention* is defined at first + use as artifact Gini over ceiling Gini on the Report partition, against a + ceiling tuned under a declared budget; the floor is reported at equal + weight. The README quick example runs through `compile_selected`, the + quickstart keeps the by-hand path, the tuning guide gains *Let the data + choose*, and the FAQ entry on distilling versus labels answers "the + pipeline selects it". API names are unchanged. +- The benchmark runs through `compile_selected`. It selected α = 0.75, 80 trees, depth 2 on Select — both pure targets scored lower — and reports 98.04% retention on Report with a 95% interval of 97.38–98.72%, against 97.9% before on a holdout that had also chosen the configuration; the artifact stands at 103.41% of a WoE logistic floor (interval 102.54–104.25%). The ceiling is now the histogram GBM (300 trees, depth 4, fixed and recorded as unsearched), so the whole run takes minutes. `results.json` gains the selected configuration, intervals, floor figures, KS, Brier and events per split, and drops `distill_spearman_train`; the selection curve is committed as `benchmarks/selection_curve.json`. The determinism check now reruns the entire protocol and requires the same configuration and the same hash. ## [0.8.0] - 2026-09-13 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fcad503..d43902d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,7 +23,7 @@ infrastructure and something a risk function could actually adopt: Smaller entry points: [#18](https://github.com/orgoca/CompileML/issues/18) (WOE compatibility docs), [#31](https://github.com/orgoca/CompileML/issues/31) (waterfall labels clipped at the canvas edge), -[#19](https://github.com/orgoca/CompileML/issues/19) (distillation cost in +[#19](https://github.com/orgoca/CompileML/issues/19) (compilation cost in expected-loss terms). Domain knowledge is as welcome as code. Several issues — segmented artifact diff --git a/README.md b/README.md index d4b4243..268afd1 100644 --- a/README.md +++ b/README.md @@ -20,55 +20,42 @@ The choice is not unique to credit. It appears wherever a model makes a decision CompileML removes that choice. -Train the strongest teacher you can: XGBoost, LightGBM, NN. CompileML projects its predictive structure into a shallow, integer-valued whitebox and packages the complete decision into one hashed artifact: score, calibrated probability, risk bands, reason codes, attribution. +Train the strongest model you can: XGBoost, LightGBM, a neural network. It sets the ceiling. CompileML then searches for the best whitebox it can compile — trained on the outcomes, on the ceiling's predictions, or on a blend — and chooses on data it will not report on. The whole decision is packaged into one hashed artifact: score, calibrated probability, risk bands, reason codes, attribution. What you get is the artifact, what its guarantees cost against that ceiling, and whether it beats the scorecard you already have. -Then the teacher is discarded. It was a means, not a deliverable, and it never reaches production. +The ceiling is a measuring instrument. It never ships. What ships instead is decision logic built from integer addition, comparison, and table lookup. It runs under CompileML's standard-library-only Python implementation, an ordinary application, a small Lambda, a batch job or you export it as standalone SQL or COBOL and run it directly where the decision already happens. No XGBoost in production, no scikit-learn, no scoring service to operate. In exported form there is no model runtime at all. -This is not a lighter way to serve the black box. The black box was used as a teacher and then compiled out of the system. +This is not a lighter way to serve the black box. The black box set the bar and was then compiled out of the system. That buys three things, and I would not trade any one of them for the other two: the artifact reproduces to the integer on any machine, it explains itself by arithmetic rather than approximation, and it lands where the decision already runs. The sections below are how each one is enforced. -The benchmark puts the cost at about 2% of the teacher's Gini. In exchange, at whitebox depth two or less, every decision reconstructs exactly from a printed scorecard table, and every explanation adds back to the production score with nothing left over. +The benchmark puts the cost at about 2% of the ceiling's Gini — 98.04% retained on rows the selection never saw, with a 95% interval of 97.38–98.72% — and the artifact out-scores a WoE logistic regression on the same rows by 3.4%. In exchange, at whitebox depth two or less, every decision reconstructs exactly from a printed scorecard table, and every explanation adds back to the production score with nothing left over. The examples in this repository come from credit, where the project began and where lenders must explain every adverse decision, reproduce it for a validator, and often run it on systems that predate Python. [Where it fits](docs/concepts/where-it-fits.md) maps the vocabulary to other domains and says where the fit is weaker. ## Quick example -Train however you want. The example below uses a strong model as a teacher and distills it into a shallow whitebox: +Train the strongest model you can. CompileML chooses the whitebox, prices it against that model, and reads the reported figure from rows it never chose on: ```python -from compileml.compile import train_whitebox -from compileml.bands import monotone_quantile_bands -from compileml.artifact import build_artifact, save_artifact +from compileml import compile_selected +from compileml.artifact import save_artifact -whitebox, fidelity = train_whitebox( - X_train, - teacher.predict_proba(X_train)[:, 1], -) - -latent = whitebox.predict(X_train).clip(0, 1) - -bands = monotone_quantile_bands( - latent, - y_train, - n_bands=10, -) - -artifact = build_artifact( - whitebox, - feature_names, - baseline=medians, - band_edges=bands, - calibration_latent=latent, - calibration_y=y_train, +result = compile_selected( + X, y, + ceiling=lambda X_fit, y_fit: XGBClassifier(**tuned).fit(X_fit, y_fit), + reference="woe", # the floor: a WoE logistic regression, fitted inside Fit reasons=REASON_DICTIONARY, ) -save_artifact(artifact, "decision.json") +result.selected # the target and configuration chosen on Select +report = result.report() # Report, read once: retention and floor ratio, with intervals +save_artifact(result.artifact, "decision.json") ``` +Every step is also a plain function — `train_whitebox`, `monotone_quantile_bands`, `build_artifact` — for when you want each one in your hands; the [quickstart](docs/quickstart.md) walks that path. + Production does not need the training stack: ```python @@ -171,7 +158,7 @@ The artifact includes a SHA-256 hash. Loaders verify it by default and reject a ## Measured performance -The committed benchmark uses deterministic synthetic credit data with 40,000 rows and 23 features. It runs on a consumer laptop through the pure-Python runtime. +The committed benchmark uses deterministic synthetic credit data with 40,000 rows and 23 features, split 60/20/20 into Fit, Select and Report. `compile_selected` chooses the target, the tree count and the depth on Select from 40 configurations; the winner is refit on Fit ∪ Select; and every retention figure below is read once from Report. It selected α = 0.75 — a blend of labels and the ceiling's cross-fitted predictions — with 80 trees at depth 2, inside a tie band of 4; both pure targets, labels alone and the ceiling alone, scored lower on Select. The ceiling is a 300-tree, depth-4 GBM with a fixed, undeclared-search configuration, recorded as such. Latency runs on a consumer laptop through the pure-Python runtime. You can reproduce every number from an editable install of the checked-out source: @@ -182,21 +169,23 @@ python benchmarks/run_benchmarks.py `results.json` records the CompileML version it measured, so a run against an older installed copy cannot pass for a measurement of the current code. -| Metric | Value | -| ------------------------------------------ | -------------------------: | -| Teacher Gini, 300-tree GBM | 0.667 | -| **Compiled integer artifact Gini** | **0.653 — 97.9% retained** | -| Band-ordinal Gini, 10 bands | 0.647 — 97.0% retained | -| Spearman correlation, teacher vs. artifact | 0.977 | -| Score + band + calibrated PD | **0.03 ms median** | -| Score + band + calibrated PD, p95 | 0.04 ms | -| Full explained decision, 120 trees | 0.62 ms median | -| Full explained decision, p95 | 0.84 ms | -| Band assignment alone | 0.2 µs | -| Artifact size | 97 KB | -| Identical hash on rebuild | Yes | - -That 2% of Gini is the price of everything above it. It is stated rather than hidden, and it is reproducible on your own data with `compileml.tune.sweep_whitebox`. +| Metric | Value | +| ---------------------------------------------- | -------------------------------------: | +| Ceiling Gini, 300-tree GBM | 0.664 | +| **Compiled integer artifact Gini** | **0.651 — 98.0% retained (97.4–98.7)** | +| Floor Gini, WoE logistic regression | 0.630 — artifact at 103.4% | +| Band-ordinal Gini, 10 bands | 0.643 — 96.7% retained | +| Spearman correlation, ceiling vs. artifact | 0.980 | +| Selected on Select, of 40 configurations | α = 0.75, 80 trees, depth 2 | +| Score + band + calibrated PD | **0.03 ms median** | +| Score + band + calibrated PD, p95 | 0.06 ms | +| Full explained decision, 80 trees | 0.70 ms median | +| Full explained decision, p95 | 1.20 ms | +| Band assignment alone | 0.4 µs | +| Artifact size | 70 KB | +| Same configuration and identical hash on rerun | Yes | + +That 2% of Gini is the price of everything above it. It is stated rather than hidden, it comes with an interval, and `compile_selected` measures it the same way on your own data. The full selection curve — every configuration scored on Select — is committed beside the results as `benchmarks/selection_curve.json`; it ranks configurations and does not describe the shipped artifact. One honest qualification: scoring is very fast; full explanation costs more. @@ -206,10 +195,10 @@ On the benchmark's 120-tree ensemble, attribution alone: | features | perturbation | per tree | tree walks, perturbation | tree walks, per tree | | -------: | -----------: | -------: | -----------------------: | -------------------: | -| 8 | 0.94 ms | 0.62 ms | 4,560 | 792 | -| 23 | 6.79 ms | 0.70 ms | 33,360 | 912 | -| 50 | 32.26 ms | 0.78 ms | 153,240 | 944 | -| 100 | 131.31 ms | 0.76 ms | 606,240 | 936 | +| 8 | 1.23 ms | 0.76 ms | 4,560 | 792 | +| 23 | 9.07 ms | 0.89 ms | 33,360 | 912 | +| 50 | 47.97 ms | 0.94 ms | 153,240 | 944 | +| 100 | 200.69 ms | 0.95 ms | 606,240 | 936 | The walk counts are exact and hold on any machine; the milliseconds belong to one laptop. Cost follows walks, and walks follow tree structure rather than width: the sweep's models are fitted to a target that uses every feature, so more of their trees split on three distinct features than the headline model's do, which is why attribution alone at 23 features here costs slightly more than the full decision in the table above. At eight features the difference between the two derivations is modest. At a hundred it is the difference between a quadratic cost and a flat one. It is exact rather than sampled, and both derivations reach identical integers on every timed row. @@ -240,7 +229,7 @@ For banding, `band_efficiency()` reports what the ladder discards — the Gini g ## What CompileML is not -CompileML is not a new training framework. Use XGBoost, LightGBM, scikit-learn, or another teacher that can be distilled into the supported whitebox representation. +CompileML is not a new training framework. Use XGBoost, LightGBM, scikit-learn, or any model that can set the ceiling for a whitebox, or be compiled into the supported representation directly. It is not a promise that your data pipelines are identical. Determinism means: @@ -325,7 +314,8 @@ It checks: 7. explanation stability; 8. reason-code coverage; 9. declared monotone directions, re-verified against the shipped trees; -10. that the artifact out-scores a reference model on the same data. +10. that the artifact out-scores a reference model on the same data; +11. that, when the artifact records how it was selected, the protocol was followed: cross-fitted soft targets, disjoint partitions, one Report read. These checks run against the compiled artifact through the same runtime used for production decisions. There is no separate notebook implementation allowed to become "almost the same" over time. @@ -413,12 +403,14 @@ Recently shipped: * exact attribution aggregated per tree, so explaining a decision no longer grows with feature count, and a fairness audit in `compileml.fairness` (0.5); * exact drift decomposition, band calibration and baseline staleness in `compileml.monitor` (0.6); * the calibrated PD and reason codes from the COBOL and SQL exports (0.7); -* retention by segment and across PD cutoff ranges, and weighting a whitebox toward the segment that pays (0.8). +* retention by segment and across PD cutoff ranges, and weighting a whitebox toward the segment that pays (0.8); +* `compile_selected`: the target and the configuration chosen on data the report never sees, the ceiling as yardstick, a provenance block that carries the reported cost, and a NumPy batch scorer (0.9). The current priorities are: -* add Java and C exporters; -* add an optional NumPy batch scorer. +* the gate experiment ([#76](https://github.com/orgoca/CompileML/issues/76)): where soft targets stop helping, and whether log-loss earns a logit-space latent; +* a logit-space latent, if that experiment supports it; +* add Java and C exporters. ## License diff --git a/benchmarks/results.json b/benchmarks/results.json index 88c69a8..63214fd 100644 --- a/benchmarks/results.json +++ b/benchmarks/results.json @@ -1,54 +1,83 @@ { "seed": 42, - "compileml": "0.5.0", + "compileml": "0.8.0", "python": "3.14.3", "machine": "Intel64 Family 6 Model 170 Stepping 4, GenuineIntel", "os": "Windows 11", "retention": { - "teacher_auc": 0.8336, - "artifact_auc": 0.8266, - "teacher_gini": 0.6673, - "artifact_gini": 0.6533, - "band_ordinal_gini": 0.6474, - "gini_retention_pct": 97.9, - "band_gini_retention_pct": 97.03, - "spearman_teacher_vs_artifact": 0.9774, - "distill_spearman_train": 0.9761, - "n_train": 30000, - "n_test": 10000, - "n_features": 23 + "protocol": "compile_selected: chosen on Select, reported once on Report", + "partition_method": "stratified", + "n_fit": 24000, + "n_select": 8000, + "n_report": 8000, + "n_features": 23, + "selected_alpha": 0.75, + "selected_trees": 80, + "selected_depth": 2, + "select_gini": 0.6587, + "n_configs": 40, + "n_tied": 4, + "eps_min": 16.1, + "ceiling_family": "sklearn.ensemble._hist_gradient_boosting.gradient_boosting.HistGradientBoostingClassifier", + "teacher_auc": 0.8323, + "artifact_auc": 0.8257, + "teacher_gini": 0.6645, + "artifact_gini": 0.6515, + "artifact_gini_ci": [ + 0.6361, + 0.6705 + ], + "artifact_ks": 0.494, + "artifact_brier": 0.15335, + "ceiling_brier": 0.15055, + "band_ordinal_gini": 0.6427, + "gini_retention_pct": 98.04, + "retention_ci": [ + 97.38, + 98.72 + ], + "band_gini_retention_pct": 96.72, + "floor_gini": 0.63, + "floor_ratio_pct": 103.41, + "floor_ratio_ci": [ + 102.54, + 104.25 + ], + "spearman_teacher_vs_artifact": 0.9801, + "n_train": 32000, + "n_test": 8000 }, "latency": { - "score_band_pd_median_ms": 0.03, - "score_band_pd_p95_ms": 0.043, - "full_explain_median_ms": 0.625, - "full_explain_p95_ms": 0.842, - "band_ladder_median_us": 0.2, + "score_band_pd_median_ms": 0.034, + "score_band_pd_p95_ms": 0.062, + "full_explain_median_ms": 0.696, + "full_explain_p95_ms": 1.197, + "band_ladder_median_us": 0.4, "explain_by_features": { "8": { - "per_tree_ms": 0.622, - "perturbation_ms": 0.938, + "per_tree_ms": 0.756, + "perturbation_ms": 1.232, "per_tree_walks": 792, "perturbation_walks": 4560, "features_split_on": 8 }, "23": { - "per_tree_ms": 0.705, - "perturbation_ms": 6.786, + "per_tree_ms": 0.889, + "perturbation_ms": 9.068, "per_tree_walks": 912, "perturbation_walks": 33360, "features_split_on": 23 }, "50": { - "per_tree_ms": 0.781, - "perturbation_ms": 32.26, + "per_tree_ms": 0.94, + "perturbation_ms": 47.965, "per_tree_walks": 944, "perturbation_walks": 153240, "features_split_on": 50 }, "100": { - "per_tree_ms": 0.759, - "perturbation_ms": 131.314, + "per_tree_ms": 0.954, + "perturbation_ms": 200.695, "per_tree_walks": 936, "perturbation_walks": 606240, "features_split_on": 88 @@ -61,15 +90,16 @@ "measures": "attribution only, per row: median ms, and tree walks", "derivations_agree": true }, - "note": "pure-Python stdlib runtime, single row, 120 depth-2 trees at p=23. full_explain is the whole decide() payload: score, band, PD, exact attribution and reason codes. Attribution is aggregated per tree, so its cost is O(trees) and independent of feature count; explain_by_features times the attribution alone by both derivations, with tree walks per row as the machine-independent measure." + "note": "pure-Python stdlib runtime, single row, 80 depth-2 trees at p=23. full_explain is the whole decide() payload: score, band, PD, exact attribution and reason codes. Attribution is aggregated per tree, so its cost is O(trees) and independent of feature count; explain_by_features times the attribution alone by both derivations, with tree walks per row as the machine-independent measure." }, "artifact": { - "size_kb": 97.2, - "n_trees": 120, - "hash": "58795bc5870fc9bd\u2026", + "size_kb": 70.2, + "n_trees": 80, + "hash": "dcabaa0dd2e99ad0\u2026", "exact_attribution": true }, "determinism": { - "rebuild_hash_identical": true + "rebuild_hash_identical": true, + "same_configuration_selected": true } } \ No newline at end of file diff --git a/benchmarks/run_benchmarks.py b/benchmarks/run_benchmarks.py index feeb5c2..d20aaba 100644 --- a/benchmarks/run_benchmarks.py +++ b/benchmarks/run_benchmarks.py @@ -2,7 +2,10 @@ Reproduces every performance and retention number stated in the README. Fully deterministic: synthetic credit-style data with a fixed seed, no -network access. Run it yourself: +network access. The whitebox is chosen by ``compile_selected`` — target, +tree count and depth selected on a partition the report never sees — and +the retention figure is read once from the Report partition. Run it +yourself: python benchmarks/run_benchmarks.py @@ -18,17 +21,17 @@ from pathlib import Path import numpy as np -from sklearn.ensemble import GradientBoostingClassifier +from sklearn.ensemble import HistGradientBoostingClassifier from sklearn.metrics import roc_auc_score -from sklearn.model_selection import train_test_split import compileml from compileml.artifact import build_artifact, save_artifact -from compileml.bands import monotone_quantile_bands, quantile_bands +from compileml.bands import quantile_bands from compileml.compile import train_whitebox from compileml.runtime import LEAF, contributions_half_micro, decide, load_artifact from compileml.runtime.bands import band_index from compileml.runtime.explain import contributions_half_micro_reference +from compileml.select import compile_selected SEED = 42 HERE = Path(__file__).resolve().parent @@ -151,67 +154,100 @@ def main() -> None: # ------------------------------------------------------------ retention n, p = 40_000, 23 X, y = make_credit_data(n, p, rng) - X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=0.25, stratify=y, random_state=SEED - ) feature_names = [f"feature_{i:02d}" for i in range(p)] - print("training teacher (sklearn GBM, 300 trees, depth 4)…") - teacher = GradientBoostingClassifier( - n_estimators=300, max_depth=4, learning_rate=0.05, random_state=SEED - ).fit(X_train, y_train) - teacher_latent_train = teacher.predict_proba(X_train)[:, 1] - teacher_latent_test = teacher.predict_proba(X_test)[:, 1] - - print("distilling whitebox (120 trees, depth 2)…") - whitebox, fidelity = train_whitebox( - X_train, teacher_latent_train, n_estimators=120, random_state=SEED - ) - latent_train = np.clip(whitebox.predict(X_train), 0, 1) - - bands = monotone_quantile_bands(latent_train, y_train, n_bands=10) - artifact = build_artifact( - whitebox, - feature_names, - np.median(X_train, axis=0), - bands, - calibration_latent=latent_train, - calibration_y=y_train, - X_sample=X_train[:500], + # The ceiling: a fixed 300-tree, depth-4 GBM. Not searched, and the + # provenance says so — retention is only comparable across artifacts when + # the effort behind the denominator is declared. + def ceiling(X_fit, y_fit): + return HistGradientBoostingClassifier( + max_iter=300, max_depth=4, learning_rate=0.05, random_state=SEED + ).fit(X_fit, y_fit) + + budget = {"trials": 1, "note": "fixed configuration, not searched"} + print("selecting the whitebox: default grid on Select, ceiling cross-fitted…") + result = compile_selected( + X, + y, + ceiling=ceiling, + ceiling_budget=budget, + reference="woe", + feature_names=feature_names, + seed=SEED, ) + unreported_hash = result.artifact["artifact_hash"] + report = result.report() + artifact = result.artifact path = HERE / "benchmark_artifact.json" save_artifact(artifact, path) artifact = load_artifact(path) + parts = result.partitions + sel = result.selected - rows_test = [[float(v) for v in row] for row in X_test] + rows_test = [[float(v) for v in row] for row in X[parts.report]] + y_test = y[parts.report] decisions = [decide(artifact, row, explain=False) for row in rows_test] - artifact_latent = np.array([d["raw_micro"] for d in decisions], dtype=float) / 1e6 + artifact_latent = np.array([d["raw_micro"] for d in decisions], dtype=float) artifact_band = np.array([d["band_idx"] for d in decisions], dtype=float) + # The ceiling refit on Fit ∪ Select is what compile_selected scored Report + # with; the same factory, seed and rows reproduce it exactly. + ceiling_scores = ceiling(X[parts.dev], y[parts.dev]).predict_proba(X[parts.report])[:, 1] def gini(score) -> float: return 2 * roc_auc_score(y_test, score) - 1 - g_teacher = gini(teacher_latent_test) - g_artifact = gini(artifact_latent) - g_band = gini(artifact_band) from scipy.stats import spearmanr + g_teacher, g_artifact = report["ceiling"]["gini"], report["artifact"]["gini"] + g_band = gini(artifact_band) results["retention"] = { - "teacher_auc": round(roc_auc_score(y_test, teacher_latent_test), 4), - "artifact_auc": round(roc_auc_score(y_test, artifact_latent), 4), + "protocol": "compile_selected: chosen on Select, reported once on Report", + "partition_method": parts.method, + "n_fit": int(len(parts.fit)), + "n_select": int(len(parts.select)), + "n_report": int(len(parts.report)), + "n_features": p, + "selected_alpha": sel["alpha"], + "selected_trees": sel["n_estimators"], + "selected_depth": sel["max_depth"], + "select_gini": round(sel["select_gini"], 4), + "n_configs": len(result.selection_curve), + "n_tied": sel["n_tied"], + "eps_min": round(artifact["metadata"]["provenance"]["eps_min"], 1), + "ceiling_family": artifact["metadata"]["provenance"]["ceiling"]["family"], + "teacher_auc": round((g_teacher + 1) / 2, 4), + "artifact_auc": round((g_artifact + 1) / 2, 4), "teacher_gini": round(g_teacher, 4), "artifact_gini": round(g_artifact, 4), + "artifact_gini_ci": [round(v, 4) for v in report["artifact_gini_ci"]], + "artifact_ks": round(report["artifact"]["ks"], 4), + "artifact_brier": round(report["artifact"]["brier"], 5), + "ceiling_brier": round(report["ceiling"]["brier"], 5), "band_ordinal_gini": round(g_band, 4), - "gini_retention_pct": round(100 * g_artifact / g_teacher, 2), + "gini_retention_pct": round(report["retention_pct"], 2), + "retention_ci": [round(v, 2) for v in report["retention_ci"]], "band_gini_retention_pct": round(100 * g_band / g_teacher, 2), + "floor_gini": round(report["floor"]["gini"], 4), + "floor_ratio_pct": round(report["floor_ratio_pct"], 2), + "floor_ratio_ci": [round(v, 2) for v in report["floor_ratio_ci"]], "spearman_teacher_vs_artifact": round( - float(spearmanr(teacher_latent_test, artifact_latent)[0]), 4 + float(spearmanr(ceiling_scores, artifact_latent)[0]), 4 ), - "distill_spearman_train": round(fidelity["spearman"], 4), - "n_train": len(X_train), - "n_test": len(X_test), - "n_features": p, + "n_train": int(len(parts.dev)), + "n_test": int(len(parts.report)), } + curve_path = HERE / "selection_curve.json" + curve_path.write_text( + json.dumps( + { + "note": "Select-partition figures, one row per configuration; " + "they rank configurations and do not describe the shipped artifact", + "rows": result.selection_curve, + }, + indent=2, + ), + encoding="utf-8", + ) # ------------------------------------------------------------ latency print("measuring latency…") @@ -271,18 +307,23 @@ def gini(score) -> float: } # ------------------------------------------------------- determinism - print("checking build determinism…") - artifact2 = build_artifact( - whitebox, - feature_names, - np.median(X_train, axis=0), - monotone_quantile_bands(latent_train, y_train, n_bands=10), - calibration_latent=latent_train, - calibration_y=y_train, - X_sample=X_train[:500], + # The whole protocol, not just the build: a second run with the same seed + # must select the same configuration and compile the same integers. + print("checking selection determinism (a second full run)…") + again = compile_selected( + X, + y, + ceiling=ceiling, + ceiling_budget=budget, + reference="woe", + feature_names=feature_names, + seed=SEED, ) results["determinism"] = { - "rebuild_hash_identical": artifact2["artifact_hash"] == artifact["artifact_hash"], + "rebuild_hash_identical": again.artifact["artifact_hash"] == unreported_hash, + "same_configuration_selected": again.selected["alpha"] == sel["alpha"] + and again.selected["n_estimators"] == sel["n_estimators"] + and again.selected["max_depth"] == sel["max_depth"], } out = HERE / "results.json" @@ -291,8 +332,22 @@ def gini(score) -> float: r, lat, art = results["retention"], results["latency"], results["artifact"] rows = [ - ("Teacher Gini", f"{r['teacher_gini']}"), - ("Artifact Gini (integer)", f"{r['artifact_gini']} ({r['gini_retention_pct']}% retention)"), + ( + "selected configuration (on Select)", + f"alpha={r['selected_alpha']}, {r['selected_trees']} trees, " + f"depth {r['selected_depth']}", + ), + ("Ceiling Gini (Report)", f"{r['teacher_gini']}"), + ( + "Artifact Gini (integer, Report)", + f"{r['artifact_gini']} ({r['gini_retention_pct']}% retention, " + f"95% CI {r['retention_ci'][0]}–{r['retention_ci'][1]})", + ), + ( + "Floor Gini (WoE logistic)", + f"{r['floor_gini']} (artifact at {r['floor_ratio_pct']}% of the floor, " + f"95% CI {r['floor_ratio_ci'][0]}–{r['floor_ratio_ci'][1]})", + ), ( "Band-ordinal Gini", f"{r['band_ordinal_gini']} ({r['band_gini_retention_pct']}% retention)", @@ -309,6 +364,10 @@ def gini(score) -> float: ("band ladder only", f"{lat['band_ladder_median_us']} µs"), ("artifact size", f"{art['size_kb']} KB"), ("rebuild hash identical", f"{results['determinism']['rebuild_hash_identical']}"), + ( + "same configuration selected on rerun", + f"{results['determinism']['same_configuration_selected']}", + ), ] print("| Metric | Value |") print("|---|---|") diff --git a/benchmarks/selection_curve.json b/benchmarks/selection_curve.json new file mode 100644 index 0000000..cb9aee1 --- /dev/null +++ b/benchmarks/selection_curve.json @@ -0,0 +1,445 @@ +{ + "note": "Select-partition figures, one row per configuration; they rank configurations and do not describe the shipped artifact", + "rows": [ + { + "alpha": 0.75, + "n_estimators": 160, + "max_depth": 2, + "monotone": false, + "eps": 16.072916666666668, + "gini": 0.6679350977765199, + "brier": 0.15084602433854638, + "tied": true, + "selected": false + }, + { + "alpha": 0.5, + "n_estimators": 160, + "max_depth": 2, + "monotone": false, + "eps": 16.072916666666668, + "gini": 0.6670018428722218, + "brier": 0.15109394717564326, + "tied": true, + "selected": false + }, + { + "alpha": 0.5, + "n_estimators": 80, + "max_depth": 2, + "monotone": false, + "eps": 32.145833333333336, + "gini": 0.6594333024659875, + "brier": 0.15247702286889575, + "tied": true, + "selected": false + }, + { + "alpha": 0.75, + "n_estimators": 80, + "max_depth": 2, + "monotone": false, + "eps": 32.145833333333336, + "gini": 0.6587119979233309, + "brier": 0.15282035861321025, + "tied": true, + "selected": true + }, + { + "alpha": 1.0, + "n_estimators": 160, + "max_depth": 2, + "monotone": false, + "eps": 16.072916666666668, + "gini": 0.6568352451604549, + "brier": 0.153333274624465, + "tied": false, + "selected": false + }, + { + "alpha": 0.0, + "n_estimators": 160, + "max_depth": 2, + "monotone": false, + "eps": 16.072916666666668, + "gini": 0.6545701913125992, + "brier": 0.15364833753233187, + "tied": false, + "selected": false + }, + { + "alpha": 1.0, + "n_estimators": 80, + "max_depth": 2, + "monotone": false, + "eps": 32.145833333333336, + "gini": 0.6536727509337565, + "brier": 0.153904950033107, + "tied": false, + "selected": false + }, + { + "alpha": 0.25, + "n_estimators": 160, + "max_depth": 2, + "monotone": false, + "eps": 16.072916666666668, + "gini": 0.6536514054765854, + "brier": 0.15393015695635562, + "tied": false, + "selected": false + }, + { + "alpha": 0.25, + "n_estimators": 80, + "max_depth": 2, + "monotone": false, + "eps": 32.145833333333336, + "gini": 0.6500285513396924, + "brier": 0.15457729014801624, + "tied": false, + "selected": false + }, + { + "alpha": 0.0, + "n_estimators": 80, + "max_depth": 2, + "monotone": false, + "eps": 32.145833333333336, + "gini": 0.6479474408945722, + "brier": 0.15494289714308984, + "tied": false, + "selected": false + }, + { + "alpha": 1.0, + "n_estimators": 40, + "max_depth": 2, + "monotone": false, + "eps": 64.29166666666667, + "gini": 0.6437247650853646, + "brier": 0.15568815292700702, + "tied": false, + "selected": false + }, + { + "alpha": 0.75, + "n_estimators": 160, + "max_depth": 1, + "monotone": false, + "eps": 48.21875, + "gini": 0.6401253336481192, + "brier": 0.1561725493532813, + "tied": false, + "selected": false + }, + { + "alpha": 1.0, + "n_estimators": 160, + "max_depth": 1, + "monotone": false, + "eps": 48.21875, + "gini": 0.6401044895943047, + "brier": 0.1563452251603485, + "tied": false, + "selected": false + }, + { + "alpha": 0.0, + "n_estimators": 40, + "max_depth": 2, + "monotone": false, + "eps": 64.29166666666667, + "gini": 0.6400275599936278, + "brier": 0.1563782431029635, + "tied": false, + "selected": false + }, + { + "alpha": 0.0, + "n_estimators": 160, + "max_depth": 1, + "monotone": false, + "eps": 48.21875, + "gini": 0.6396596015591063, + "brier": 0.15615352522867862, + "tied": false, + "selected": false + }, + { + "alpha": 0.5, + "n_estimators": 40, + "max_depth": 2, + "monotone": false, + "eps": 64.29166666666667, + "gini": 0.6395275175892294, + "brier": 0.15629586600394765, + "tied": false, + "selected": false + }, + { + "alpha": 0.75, + "n_estimators": 40, + "max_depth": 2, + "monotone": false, + "eps": 64.29166666666667, + "gini": 0.6394424222767494, + "brier": 0.1563252805297, + "tied": false, + "selected": false + }, + { + "alpha": 0.5, + "n_estimators": 160, + "max_depth": 1, + "monotone": false, + "eps": 48.21875, + "gini": 0.639304393095647, + "brier": 0.15624304607158926, + "tied": false, + "selected": false + }, + { + "alpha": 0.25, + "n_estimators": 160, + "max_depth": 1, + "monotone": false, + "eps": 48.21875, + "gini": 0.6391223120482357, + "brier": 0.1566332728378495, + "tied": false, + "selected": false + }, + { + "alpha": 1.0, + "n_estimators": 80, + "max_depth": 1, + "monotone": false, + "eps": 96.4375, + "gini": 0.63833890511844, + "brier": 0.15639526028214226, + "tied": false, + "selected": false + }, + { + "alpha": 0.75, + "n_estimators": 80, + "max_depth": 1, + "monotone": false, + "eps": 96.4375, + "gini": 0.6382253730727487, + "brier": 0.15657505145979989, + "tied": false, + "selected": false + }, + { + "alpha": 0.25, + "n_estimators": 40, + "max_depth": 2, + "monotone": false, + "eps": 64.29166666666667, + "gini": 0.6379883525432897, + "brier": 0.15660266204979528, + "tied": false, + "selected": false + }, + { + "alpha": 0.0, + "n_estimators": 80, + "max_depth": 1, + "monotone": false, + "eps": 96.4375, + "gini": 0.6374670363107715, + "brier": 0.15647856669405988, + "tied": false, + "selected": false + }, + { + "alpha": 0.5, + "n_estimators": 80, + "max_depth": 1, + "monotone": false, + "eps": 96.4375, + "gini": 0.6371793740423195, + "brier": 0.1567008349466509, + "tied": false, + "selected": false + }, + { + "alpha": 0.25, + "n_estimators": 80, + "max_depth": 1, + "monotone": false, + "eps": 96.4375, + "gini": 0.6370687787877154, + "brier": 0.15678138121589688, + "tied": false, + "selected": false + }, + { + "alpha": 1.0, + "n_estimators": 20, + "max_depth": 2, + "monotone": false, + "eps": 128.58333333333334, + "gini": 0.6334086775443499, + "brier": 0.15761127095716437, + "tied": false, + "selected": false + }, + { + "alpha": 0.75, + "n_estimators": 20, + "max_depth": 2, + "monotone": false, + "eps": 128.58333333333334, + "gini": 0.6319947917084501, + "brier": 0.15763309949442764, + "tied": false, + "selected": false + }, + { + "alpha": 0.5, + "n_estimators": 20, + "max_depth": 2, + "monotone": false, + "eps": 128.58333333333334, + "gini": 0.6307860514743551, + "brier": 0.15792241482547287, + "tied": false, + "selected": false + }, + { + "alpha": 0.75, + "n_estimators": 40, + "max_depth": 1, + "monotone": false, + "eps": 192.875, + "gini": 0.6307662102272531, + "brier": 0.15793447224335902, + "tied": false, + "selected": false + }, + { + "alpha": 1.0, + "n_estimators": 40, + "max_depth": 1, + "monotone": false, + "eps": 192.875, + "gini": 0.630586994341878, + "brier": 0.15779925963265237, + "tied": false, + "selected": false + }, + { + "alpha": 0.0, + "n_estimators": 20, + "max_depth": 2, + "monotone": false, + "eps": 128.58333333333334, + "gini": 0.6304127924900664, + "brier": 0.1580834779367335, + "tied": false, + "selected": false + }, + { + "alpha": 0.0, + "n_estimators": 40, + "max_depth": 1, + "monotone": false, + "eps": 192.875, + "gini": 0.6298684117031557, + "brier": 0.15798007717872828, + "tied": false, + "selected": false + }, + { + "alpha": 0.25, + "n_estimators": 20, + "max_depth": 2, + "monotone": false, + "eps": 128.58333333333334, + "gini": 0.629783459648777, + "brier": 0.15795829911252288, + "tied": false, + "selected": false + }, + { + "alpha": 0.25, + "n_estimators": 40, + "max_depth": 1, + "monotone": false, + "eps": 192.875, + "gini": 0.6296182114283291, + "brier": 0.15801733979712074, + "tied": false, + "selected": false + }, + { + "alpha": 0.5, + "n_estimators": 40, + "max_depth": 1, + "monotone": false, + "eps": 192.875, + "gini": 0.6293873509972481, + "brier": 0.15804886228367776, + "tied": false, + "selected": false + }, + { + "alpha": 1.0, + "n_estimators": 20, + "max_depth": 1, + "monotone": false, + "eps": 385.75, + "gini": 0.6110686510014889, + "brier": 0.16080303857055062, + "tied": false, + "selected": false + }, + { + "alpha": 0.75, + "n_estimators": 20, + "max_depth": 1, + "monotone": false, + "eps": 385.75, + "gini": 0.6099104808773357, + "brier": 0.1614319145787316, + "tied": false, + "selected": false + }, + { + "alpha": 0.5, + "n_estimators": 20, + "max_depth": 1, + "monotone": false, + "eps": 385.75, + "gini": 0.6091987029984494, + "brier": 0.16125078760049613, + "tied": false, + "selected": false + }, + { + "alpha": 0.25, + "n_estimators": 20, + "max_depth": 1, + "monotone": false, + "eps": 385.75, + "gini": 0.6090312342774231, + "brier": 0.16124438045726863, + "tied": false, + "selected": false + }, + { + "alpha": 0.0, + "n_estimators": 20, + "max_depth": 1, + "monotone": false, + "eps": 385.75, + "gini": 0.6069492642836924, + "brier": 0.162079088190072, + "tied": false, + "selected": false + } + ] +} \ No newline at end of file diff --git a/docs/ARTIFACT_SPEC.md b/docs/ARTIFACT_SPEC.md index 929dee6..5ce59e3 100644 --- a/docs/ARTIFACT_SPEC.md +++ b/docs/ARTIFACT_SPEC.md @@ -186,7 +186,7 @@ Notes: source framework compares in float32 internally (XGBoost hist trees): the compiler stores float32-adjusted thresholds, and quantizing the inputs makes every conforming runtime reproduce the source model's routing exactly. - `"float64"` (the default, and always the case for distilled whiteboxes) + `"float64"` (the default, and always the case for whiteboxes from `train_whitebox`) means inputs are used as given. - Accumulation order is irrelevant (integer addition is associative), so parallel or reordered implementations remain bit-identical. diff --git a/docs/concepts/attribution.md b/docs/concepts/attribution.md index 8ab16d1..552f8cd 100644 --- a/docs/concepts/attribution.md +++ b/docs/concepts/attribution.md @@ -60,10 +60,10 @@ From the committed benchmark, attribution alone on a 120-tree ensemble: | features | perturbation | per tree | tree walks, perturbation | tree walks, per tree | |---|---|---|---|---| -| 8 | 0.94 ms | 0.62 ms | 4,560 | 792 | -| 23 | 6.79 ms | 0.70 ms | 33,360 | 912 | -| 50 | 32.26 ms | 0.78 ms | 153,240 | 944 | -| 100 | 131.31 ms | 0.76 ms | 606,240 | 936 | +| 8 | 1.23 ms | 0.76 ms | 4,560 | 792 | +| 23 | 9.07 ms | 0.89 ms | 33,360 | 912 | +| 50 | 47.97 ms | 0.94 ms | 153,240 | 944 | +| 100 | 200.69 ms | 0.95 ms | 606,240 | 936 | Flat, not merely faster. The walk counts are exact and hold on any machine; the milliseconds are one laptop's. The benchmark's target spreads signal over @@ -87,7 +87,7 @@ attributions rather than sample estimates with selection effects. This is why `decide()` defaults to `explain=True`. Batch re-explanation used to be the place the cost hurt: 10M accounts at -8.4 ms is roughly 23 CPU-hours, and it grew with p². At 0.62 ms it is under +8.4 ms is roughly 23 CPU-hours, and it grew with p². At 0.70 ms it is under two, and it no longer grows with feature count. That is what makes reason-code emission in the SQL export tractable rather than impractical. Live latency was never the constraint. diff --git a/docs/concepts/bands.md b/docs/concepts/bands.md index 9de07e1..78ba4ec 100644 --- a/docs/concepts/bands.md +++ b/docs/concepts/bands.md @@ -43,7 +43,7 @@ five. There is a test pinning each behavior. !!! note "Strictness is a knob" `eps_auc` controls how much residual within-band ranking you tolerate. - Distilled whitebox latents are naturally plateaued (a depth-2, 120-tree + Whitebox latents are naturally plateaued (a depth-2, 120-tree model takes finitely many values), which is the intended input. A smooth, steadily-sloped latent may legitimately support only one band under a strict `eps_auc` — that is the method telling you band boundaries would be diff --git a/docs/concepts/where-it-fits.md b/docs/concepts/where-it-fits.md index 077d4cf..0daa10b 100644 --- a/docs/concepts/where-it-fits.md +++ b/docs/concepts/where-it-fits.md @@ -54,7 +54,7 @@ read them this way: | cutoff range | where the decision threshold will sit | risk appetite | review capacity | escalation threshold | contact threshold | | reason code | a top driver of one decision | adverse-action reason | why the alert fired | why the case was flagged | retention driver | | `risk_increasing` / `risk_decreasing` | pushes toward / away from the positive outcome | raises / lowers default risk | raises / lowers fraud risk | raises / lowers event risk | raises / lowers churn risk | -| teacher retention | how much of the teacher's ranking survives compilation | same | same | same | same | +| retention | how much of the ceiling model's (the teacher's) ranking survives compilation | same | same | same | same | If you use CompileML outside credit, what compiling cost on your data is exactly what [Show and tell](https://github.com/orgoca/CompileML/discussions/categories/show-and-tell) diff --git a/docs/faq.md b/docs/faq.md index 54eef98..b8f6c86 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -67,19 +67,23 @@ tell you the artifact is being beaten by a logistic regression on the same data — which does happen, particularly at depth 2 where capacity is the binding constraint. Fit the other side of the comparison with `compileml.reference.fit_reference` (or pass your champion scorecard's Gini -as a plain float) and every sweep row carries the floor beside the ceiling. -Validation check 10 applies the same comparison to a built artifact. See +as a plain float) and every sweep row carries the floor beside the ceiling; +`compile_selected` reports both at equal weight, with intervals, and refuses +to build an artifact that does not clear the floor. Validation check 10 +applies the same comparison to a built artifact. See [the tuning guide](howto/tuning.md#the-floor-what-should-the-whitebox-beat). ## Should I distil from a teacher, or train on labels directly? -Measure it rather than assuming. `train_whitebox` takes any target, and -`sweep_whitebox(alpha_grid=...)` sweeps the blend -`alpha * y + (1 - alpha) * teacher_latent` from pure distillation to pure -labels. At a small tree-and-depth budget, regressing onto a teacher's -probabilities can spend capacity on teacher noise instead of outcome; whether -that happens on your data is an empirical question. Sweep the target -alongside capacity, not on its own — the two are easy to confuse. +Let the pipeline select it. `compile_selected` trains candidates on the +labels, on the ceiling's cross-fitted predictions, and on blends of the two, +and picks on rows it will not report on. On a 2.9M-row credit portfolio the +whitebox trained on labels beat the distilled one, so distillation is no +longer the default anywhere; soft targets earn their place when events are +scarce and the whitebox would otherwise fit label noise. Sweeping by hand is +still possible with `sweep_whitebox(alpha_grid=...)` — sweep the target +alongside capacity, since the two are easy to confuse, and use out-of-fold +ceiling predictions, since in-sample ones partly restate the labels. ## Why not just use SHAP? @@ -98,11 +102,11 @@ PMML, ONNX and m2cgen solve related but different problems, and each is the better choice when its problem is yours. PMML is mature, widely understood by validators, and its Scorecard model already carries points and reason codes. ONNX is the broad inference standard, and the natural choice for a model such -as a neural network you do not want to distill. m2cgen turns a trained model +as a neural network you do not want to compile through a whitebox. m2cgen turns a trained model into readable code in many languages, including ones CompileML does not export to. -All three run the model you trained. CompileML runs a distilled one — a +All three run the model you trained. CompileML runs a compiled whitebox — a depth-2 whitebox that gives up some discrimination, about 2% of Gini on the committed benchmark — in exchange for a different guarantee. It targets the **decision**, not just the scorer: integer arithmetic fixed in the artifact @@ -111,13 +115,14 @@ SQL and COBOL agree to the integer; calibration, bands and reason codes in one hashed document; and attributions for the whole tree ensemble that add back to the score exactly. If you need to run a model elsewhere, use one of those tools. If you need the deployed decision itself to be deterministic and -auditable, and can afford the distillation, that is the problem CompileML is +auditable, and can afford the compression, that is the problem CompileML is designed to solve. ## Can I compile my XGBoost classifier directly? Directly compiled models must emit a latent in [0, 1] — a classifier's raw -margin lives in log-odds space and will be clamped into nonsense. Distill it: +margin lives in log-odds space and will be clamped into nonsense. Train a +whitebox on its probabilities instead: `train_whitebox(X, model.predict_proba(X)[:, 1])`. Regressors on probability-like targets compile directly, and the build warns when sample latents fall outside range. @@ -125,7 +130,7 @@ latents fall outside range. ## What about neural networks? Same route: any model that produces a probability-like latent can teach a -whitebox. The artifact never contains the network — it contains the distilled +whitebox. The artifact never contains the network — it contains the compiled trees, with the retention measured and recorded. ## What happens when I retrain or recalibrate? @@ -164,7 +169,7 @@ labels. It is still more work than scoring, but no longer dramatically so, and no longer quadratic in feature count. Attribution is aggregated per tree, which makes the cost `O(trees)` and independent of `p` — on the committed -benchmark's 120-tree ensemble, 0.6–0.8 ms per row whether the model has +benchmark's 120-tree ensemble, 0.8–1.0 ms per row whether the model has 8 features or 100. More trees cost proportionally more; more features do not. That is real-time for credit decisioning, which is why diff --git a/docs/howto/tuning.md b/docs/howto/tuning.md index 76b76cf..5f4cc80 100644 --- a/docs/howto/tuning.md +++ b/docs/howto/tuning.md @@ -17,7 +17,96 @@ size — what degrades above depth 2 is *explainability*, and only that. **Spend on trees; be stingy with depth.** -## Finding the tree count +## Let the data choose: `compile_selected` + +Two things used to be assumed. That the whitebox should learn from the +teacher's probabilities — and on a 2.9M-row credit portfolio the whitebox +trained on labels beat the distilled one, with bootstrap intervals excluding +zero, and a WoE logistic regression beat both compiled models ([#39](https://github.com/orgoca/CompileML/issues/39)). +And that one holdout could both choose a configuration and report its +retention — which leaks: the figure chosen on a holdout is optimistic on it. + +`compile_selected` replaces both assumptions with a protocol: + +```python +from compileml import compile_selected + +result = compile_selected( + X, y, + ceiling=lambda X_fit, y_fit: XGBClassifier(**tuned).fit(X_fit, y_fit), + ceiling_budget={"trials": 200, "cv": "5-fold on fit"}, # recorded, so retention is comparable + reference="woe", # or your champion scorecard's Gini + date=application_date, # Report becomes the latest slice + group=applicant_id, # one applicant, one partition +) +result.selection_curve # every configuration, scored on Select +result.selected # alpha, trees, depth, and the tie band it won inside +report = result.report() # Report, read once +``` + +**Three roles.** The *ceiling* — the teacher — is the strongest model you can +train under a declared budget. It prices compilation, it may supply soft +targets, and it never ships. The *floor* is a WoE logistic regression fitted +inside Fit, or a champion scorecard's Gini. The *candidate* is the depth-≤2 +whitebox, one per configuration. + +**Three partitions.** *Fit* trains everything, including calibration and +band edges. *Select* chooses the target, the tree count, the depth and the +monotone set. *Report* is read once, for the published figure. The default +split is 60/20/20 stratified; `date=` makes Report the latest slice, and +`group=` keeps each applicant in one partition, so a repeat applicant cannot +sit in Fit and Report at once. + +**Soft targets are cross-fitted.** A ceiling scoring its own training rows +has partly memorised their labels, so in-sample predictions restate the +label and dilute the soft-target arm. The factory is called once, its +configuration is frozen, and that configuration is refit K times for +out-of-fold predictions: one search plus K fits, never K searches. + +**The candidate is scored as an artifact.** Every configuration is +quantized, calibrated and banded on Fit and scored on Select as an integer +artifact, so quantization and banding are priced, not just the tree fit. The +tree axis is free: the first 20 trees of a 160-tree fit *are* the 20-tree +model, so one fit per (α, depth) covers every tree count. + +**One-SE-simplest.** The best configuration's Select metric is bootstrapped +for its standard error; everything within one SE is tied; among the tied, +α = 1 wins, then fewer trees, then lower depth. The α preference is a +governance prior — an artifact trained on labels has no training dependency +on the ceiling — and provenance records it as the tie rule. On large data the +SE is tiny and the data decides; on small data the prior decides. + +**The floor gates.** If the selected configuration does not beat the floor +on Select, you get a result and no artifact, unless `allow_below_floor=True`. +Retention against the ceiling and the ratio against the floor are reported at +equal weight: the first prices the guarantees, the second says whether +adopting them is worth it. + +**The winner is refit on Fit ∪ Select** before Report is read. The selection +curve therefore ranks *configurations*; the Report figure is the only number +that describes the artifact that ships. `result.report()` writes that figure +into the artifact's hash-covered provenance block and refuses to run twice — +hygiene, not proof, and it says so. + +**Small data.** Below `min_select_events` (300) in Select, selection runs as +nested cross-validation over Fit ∪ Select, with a warning; Report stays held +out. + +**Events per split.** Every configuration reports `eps`, the positives in +Fit per split in the ensemble. Soft targets help when events are scarce and +the whitebox would otherwise fit label noise; the boundary measured so far — +soft targets winning below roughly 10 events per split — came from one +in-sample sweep, is a lower bound, and is being replaced by +[#76](https://github.com/orgoca/CompileML/issues/76). It is a diagnostic, never +a selection input. + +**Cost.** The search uses the histogram backend, which at 300k rows fitted +in a second where the classic backend took five minutes; the tree axis costs +nothing; Select is scored with the NumPy batch scorer. The ceiling's refits +dominate: budget for about K + 2 fits of your strongest model beyond the one +you would train anyway. + +## Finding the tree count by hand ```python from compileml.tune import sweep_whitebox @@ -35,11 +124,16 @@ Each row reports holdout Gini and retention versus the teacher, Spearman rank agreement, `exact_attribution`, the quantized model's JSON size, and a *measured* per-row exact-explanation cost. Read it like a cost curve: retention climbs steeply, then plateaus; pick the elbow. In the repository benchmark, -120 trees at depth 2 retained 97.9% of a 300-tree teacher's Gini — beyond the +the selected configuration — α = 0.75, 80 trees, depth 2 — retained 98.04% of a +300-tree ceiling's Gini on Report (95% interval 97.38–98.72%); 160 trees scored +basis points higher on Select and lost the tie to the smaller model. Beyond the plateau you pay linear size and explain time for basis points of fidelity. -Always sweep on a holdout (`X_val=`) — in-sample retention flatters every -configuration, and the output flags `in_sample: true` when you didn't. +A holdout (`X_val=`) is better than in-sample, where retention flatters +every configuration — but a holdout used both to choose a configuration and to +report its retention leaks, and `sweep_whitebox` now warns when it sees one. +Choose on it; report on rows never used to choose. That is what +`compile_selected` does for you. ## Choosing depth: one table, one story @@ -423,15 +517,17 @@ rows = sweep_whitebox(X, teacher, y, reference=0.8515, ...) The same argument goes to `validate_artifact(reference=...)` as check 10, advisory by default and gateable with `require_reference_floor=True`. -## Choosing the target: distillation is a choice, not a given +## Choosing the target: a selected parameter, not a given -`train_whitebox` regresses onto whatever target you hand it. Distilling from -a teacher's probabilities is the default, but at a depth-2 budget a soft -target can spend capacity fitting the teacher's noise rather than the -outcome — and the labels are right there. +`train_whitebox` regresses onto whatever target you hand it: the labels, the +ceiling's out-of-fold probabilities, or a blend `alpha * y + (1 - alpha) * +ceiling`. Which one wins is a property of your data, not of the method — on +the portfolio above the labels won outright — so `compile_selected` searches +α alongside capacity and picks on Select. Pure distillation, `alpha_grid=(0.0,)`, +was the old default of `sweep_whitebox`; it now warns when the grid is not +passed, and the default becomes `(0.0, 0.5, 1.0)` in 1.0. -`alpha_grid` sweeps it, training each configuration on -`alpha * y + (1 - alpha) * teacher_latent`: +By hand, `alpha_grid` sweeps it: ```python rows = sweep_whitebox( @@ -449,12 +545,17 @@ endpoints bracket the answer: soft targets sometimes regularize, so an interior blend can win. That is the argument for sweeping rather than asserting either end. -To drop the teacher entirely, pass `teacher_latent=None`; `alpha_grid` is -forced to `(1.0,)` and the teacher columns come back `None`. +To drop the ceiling entirely, pass `teacher_latent=None`; `alpha_grid` is +forced to `(1.0,)` and the teacher columns come back `None`. Whatever you +sweep by hand, feed `sweep_whitebox` the ceiling's *out-of-fold* predictions: +in-sample ones partly restate the labels and understate what a soft target +does. ## Defaults, for the impatient +`compile_selected(X, y, ceiling=..., reference="woe")` with its default +grid — α ∈ {0, 0.25, 0.5, 0.75, 1}, trees {20, 40, 80, 160}, depth {1, 2} — is +the measured path, and the repository benchmark runs through it. By hand, `train_whitebox(n_estimators=30, max_depth=2)` and `n_bands=10` are sane -starting points, proven in the repository's own benchmark and examples. The -sweeps are for when "sane" needs to become "measured" — which, in a model -governance file, it eventually does. +starting points. The sweeps are for when "sane" needs to become "measured" — +which, in a model governance file, it eventually does. diff --git a/docs/howto/validate.md b/docs/howto/validate.md index 234849c..8226aa9 100644 --- a/docs/howto/validate.md +++ b/docs/howto/validate.md @@ -24,7 +24,7 @@ Or gate a pipeline on the CLI's exit code: compileml validate decision.json --csv holdout.csv --y-col DEFAULT --require-reasons ``` -## The ten checks +## The eleven checks | # | Check | What it proves | Needs | |---|---|---|---| @@ -38,6 +38,7 @@ compileml validate decision.json --csv holdout.csv --y-col DEFAULT --require-rea | 8 | reason coverage | dictionary coverage of feature names; optional hard gate | nothing | | 9 | monotone constraints | declared directions re-verified against the shipped integer trees ([spec §3.1](../ARTIFACT_SPEC.md)) | nothing | | 10 | reference floor | the artifact out-scores a reference model on the same data — the floor teacher retention cannot supply | X + y + reference | +| 11 | selection hygiene | when the artifact carries a provenance block from `compile_selected`: soft targets cross-fitted, partitions disjoint, Report read once; advisory unless `require_selection_hygiene=True` | nothing | Checks lacking inputs **skip** (reported as skipped, not passed silently); checks 1 and 8 always run, and check 9 runs whenever the artifact declares diff --git a/docs/index.md b/docs/index.md index e818a0a..05b0b7f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -22,6 +22,7 @@ model ──compile──► decision.json ──runs on──► stdlib Pyt ## Where to start - New here → [Quickstart](quickstart.md) +- Choosing the whitebox → [Let the data choose](howto/tuning.md) - Not in lending? → [Where it fits](concepts/where-it-fits.md) - Want the exact rules → [Artifact specification](ARTIFACT_SPEC.md) - "Why should I believe the determinism claim?" → [Determinism](concepts/determinism.md) diff --git a/docs/quickstart.md b/docs/quickstart.md index adadbe7..33313db 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -8,12 +8,28 @@ Ten minutes from a fitted model to a verified, deployable decision artifact. pip install compileml ``` -## 1. Train a teacher, distill a whitebox +## 1. Train a ceiling model, compile a whitebox -Any strong model can be the teacher — its only job is to produce a -probability-like latent. The whitebox is a small depth-2 gradient-boosted -regressor fitted to that latent; depth 2 is what makes attribution *exact* -(see [Exact attribution](concepts/attribution.md)). +Any strong model can set the ceiling: it prices what compilation costs, and it +may supply the whitebox's training target. The whitebox is a small depth-2 +gradient-boosted regressor; depth 2 is what makes attribution *exact* (see +[Exact attribution](concepts/attribution.md)). + +The recommended path is one call, which selects the target (labels, the +ceiling's predictions, or a blend), the tree count and the depth on rows it +will not report on, and refits the winner: + +```python +from compileml import compile_selected + +result = compile_selected(X, y, ceiling=lambda Xf, yf: teacher_factory(Xf, yf), reference="woe") +artifact = result.artifact +report = result.report() # read once +``` + +See [the tuning guide](howto/tuning.md) for what it does and why. The rest of +this page walks the same steps by hand, which is worth doing once to see what +each one produces. ```python import numpy as np diff --git a/docs/reference/api.md b/docs/reference/api.md index f8913e6..8437692 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -40,6 +40,18 @@ ::: compileml.tune.retention_by_segment +## Selection + +::: compileml.select.compile_selected + +::: compileml.select.SelectionResult + +::: compileml.select.make_partitions + +::: compileml.select.default_grid + +::: compileml.select.choose + ## Scorecard ::: compileml.scorecard.build_scorecard @@ -68,6 +80,10 @@ ::: compileml.runtime.decide +## Batch scoring (learning side) + +::: compileml.batch.score_batch + ## Fairness ::: compileml.fairness.FairnessAudit diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 872d7d8..71d3c46 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -48,9 +48,10 @@ handled per the artifact's missing policy. ```bash compileml validate decision.json --csv holdout.csv --y-col DEFAULT --require-reasons +compileml validate decision.json --csv holdout.csv --y-col DEFAULT --require-selection-hygiene ``` -Runs the [ten-check framework](../howto/validate.md); prints the full +Runs the [eleven-check framework](../howto/validate.md); prints the full evidence report as JSON; exits non-zero if any check fails — suitable as a CI deployment gate. From 12da0e5730bc572d5082af41b84eaa93757e31b9 Mon Sep 17 00:00:00 2001 From: Carlos Ortiz Date: Sun, 27 Sep 2026 10:22:56 -0600 Subject: [PATCH 3/5] Say why the latency figures moved Every millisecond figure in the rerun is 1.2-1.5x slower than the 0.5.1 run on identical models with identical tree-walk counts: the reference laptop's state on the day, not the code. The changelog says so; the transcribed copies follow the committed file as they must. --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 847fa34..33bb64c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,12 @@ inside each artifact (`schema_version`). pipeline selects it". API names are unchanged. - The benchmark runs through `compile_selected`. It selected α = 0.75, 80 trees, depth 2 on Select — both pure targets scored lower — and reports 98.04% retention on Report with a 95% interval of 97.38–98.72%, against 97.9% before on a holdout that had also chosen the configuration; the artifact stands at 103.41% of a WoE logistic floor (interval 102.54–104.25%). The ceiling is now the histogram GBM (300 trees, depth 4, fixed and recorded as unsearched), so the whole run takes minutes. `results.json` gains the selected configuration, intervals, floor figures, KS, Brier and events per split, and drops `distill_spearman_train`; the selection curve is committed as `benchmarks/selection_curve.json`. The determinism check now reruns the entire protocol and requires the same configuration and the same hash. + The latency block was re-measured in the same run and every millisecond figure moved by + 1.2–1.5× on models identical to 0.5.1's, with identical tree-walk counts, so the change is + the reference laptop's state on the day, not the code. The transcribed copies in the README, + the attribution page and the FAQ follow the committed file, as they must; the walk counts + are the machine-independent measure and are unchanged. + ## [0.8.0] - 2026-09-13 What compilation cost where decisions are made — and what to do about it. From b6040ba90f6345019ecf78a973e30e20c65f9da6 Mon Sep 17 00:00:00 2001 From: Carlos Ortiz Date: Sun, 27 Sep 2026 10:24:09 -0600 Subject: [PATCH 4/5] Write the transcribed benchmark figures from results.json, never by hand benchmarks/sync_docs.py copies the committed figures into the README table, the attribution page's cost table and the two sentences that quote a latency. Each target must match exactly once or the script stops, and it is idempotent. RELEASING.md gains a section on refreshing the benchmark on an idle machine. --- benchmarks/sync_docs.py | 137 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 benchmarks/sync_docs.py diff --git a/benchmarks/sync_docs.py b/benchmarks/sync_docs.py new file mode 100644 index 0000000..ddc2fcc --- /dev/null +++ b/benchmarks/sync_docs.py @@ -0,0 +1,137 @@ +"""Copy the committed benchmark figures into the documents that transcribe them. + +benchmarks/results.json is generated; the README table, the attribution page's +cost table and two sentences in the docs are transcribed from it. A transcribed +number that has drifted from the file is worse than no number, so this script +is the only way those copies get written. Run it after run_benchmarks.py: + + python benchmarks/run_benchmarks.py + python benchmarks/sync_docs.py + +It is idempotent, and it refuses to guess: every target is matched exactly +once or the script stops. +""" + +from __future__ import annotations + +import io +import json +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +RESULTS = ROOT / "benchmarks" / "results.json" +README = ROOT / "README.md" +ATTRIBUTION = ROOT / "docs" / "concepts" / "attribution.md" +FAQ = ROOT / "docs" / "faq.md" + +SWEEP_ROW = r"^\| {p} \| [0-9.]+ ms \| [0-9.]+ ms \| [0-9,]+ \| [0-9,]+ \|$" + + +def read(path: Path) -> str: + return io.open(path, encoding="utf-8").read() + + +def write(path: Path, text: str) -> None: + io.open(path, "w", encoding="utf-8", newline="\n").write(text) + + +def replace_once(text: str, pattern: str, replacement: str, where: str, flags=re.M) -> str: + matches = re.findall(pattern, text, flags) + if len(matches) != 1: + raise SystemExit( + f"{where}: expected exactly one match for {pattern!r}, found {len(matches)}" + ) + return re.sub(pattern, replacement, text, count=1, flags=flags) + + +def readme_table(res: dict) -> str: + r, lat, art, det = res["retention"], res["latency"], res["artifact"], res["determinism"] + lo, hi = r["retention_ci"] + rows = [ + ("Ceiling Gini, 300-tree GBM", f"{r['teacher_gini']:.3f}"), + ( + "**Compiled integer artifact Gini**", + f"**{r['artifact_gini']:.3f} — {r['gini_retention_pct']:.1f}% retained ({lo:.1f}–{hi:.1f})**", + ), + ( + "Floor Gini, WoE logistic regression", + f"{r['floor_gini']:.3f} — artifact at {r['floor_ratio_pct']:.1f}%", + ), + ( + "Band-ordinal Gini, 10 bands", + f"{r['band_ordinal_gini']:.3f} — {r['band_gini_retention_pct']:.1f}% retained", + ), + ("Spearman correlation, ceiling vs. artifact", f"{r['spearman_teacher_vs_artifact']:.3f}"), + ( + f"Selected on Select, of {r['n_configs']} configurations", + f"α = {r['selected_alpha']}, {r['selected_trees']} trees, depth {r['selected_depth']}", + ), + ("Score + band + calibrated PD", f"**{lat['score_band_pd_median_ms']:.2f} ms median**"), + ("Score + band + calibrated PD, p95", f"{lat['score_band_pd_p95_ms']:.2f} ms"), + ( + f"Full explained decision, {art['n_trees']} trees", + f"{lat['full_explain_median_ms']:.2f} ms median", + ), + ("Full explained decision, p95", f"{lat['full_explain_p95_ms']:.2f} ms"), + ("Band assignment alone", f"{lat['band_ladder_median_us']:.1f} µs"), + ("Artifact size", f"{art['size_kb']:.0f} KB"), + ( + "Same configuration and identical hash on rerun", + "Yes" if det["rebuild_hash_identical"] and det["same_configuration_selected"] else "No", + ), + ] + w = max(len(a) for a, _ in rows) + v = max(len(b) for _, b in rows) + lines = [f"| {'Metric':<{w}} | {'Value':>{v}} |", f"| {'-' * w} | {'-' * (v - 1)}: |"] + lines += [f"| {a:<{w}} | {b:>{v}} |" for a, b in rows] + return "\n".join(lines) + "\n" + + +def sweep_rows(text: str, sweep: dict, where: str) -> str: + for p in ("8", "23", "50", "100"): + v = sweep[p] + row = ( + f"| {p} | {v['perturbation_ms']:.2f} ms | {v['per_tree_ms']:.2f} ms | " + f"{v['perturbation_walks']:,} | {v['per_tree_walks']:,} |" + ) + text = replace_once(text, SWEEP_ROW.format(p=p), row, f"{where} sweep row p={p}") + return text + + +def main() -> None: + res = json.loads(read(RESULTS)) + lat = res["latency"] + sweep = lat["explain_by_features"] + per_tree = [sweep[p]["per_tree_ms"] for p in ("8", "23", "50", "100")] + + readme = read(README) + readme = replace_once( + readme, r"\| Metric[^\n]*\n\|[- |:]+\n(?:\|[^\n]*\n)+", readme_table(res), "README table" + ) + readme = sweep_rows(readme, sweep, "README") + write(README, readme) + + attribution = sweep_rows(read(ATTRIBUTION), sweep, "attribution.md") + attribution = replace_once( + attribution, + r"At [0-9.]+ ms it is under", + f"At {lat['full_explain_median_ms']:.2f} ms it is under", + "attribution.md batch sentence", + ) + write(ATTRIBUTION, attribution) + + faq = replace_once( + read(FAQ), + r"[0-9.]+–[0-9.]+ ms per row whether the model has", + f"{min(per_tree):.1f}–{max(per_tree):.1f} ms per row whether the model has", + "faq.md cost range", + ) + write(FAQ, faq) + print( + "synced README.md, docs/concepts/attribution.md, docs/faq.md from benchmarks/results.json" + ) + + +if __name__ == "__main__": + main() From 14cbc0b95e0cf2fc01abecdac8cf9a7bfce42944 Mon Sep 17 00:00:00 2001 From: Carlos Ortiz Date: Sun, 27 Sep 2026 10:25:03 -0600 Subject: [PATCH 5/5] Lint sync_docs.py and point the README and RELEASING at it The previous commit landed the script before ruff had passed. Builtin open with context managers, one wrapped line, and the two mentions: the README says its table is written by the script, never by hand, and RELEASING.md gains a section on refreshing the benchmark on an idle machine. --- README.md | 2 +- RELEASING.md | 15 +++++++++++++++ benchmarks/sync_docs.py | 15 +++++++-------- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 268afd1..ab951ac 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ pip install -e . python benchmarks/run_benchmarks.py ``` -`results.json` records the CompileML version it measured, so a run against an older installed copy cannot pass for a measurement of the current code. +`results.json` records the CompileML version it measured, so a run against an older installed copy cannot pass for a measurement of the current code. The table below and the cost figures elsewhere in the docs are written from that file by `python benchmarks/sync_docs.py`, never by hand. | Metric | Value | | ---------------------------------------------- | -------------------------------------: | diff --git a/RELEASING.md b/RELEASING.md index c1540b1..97b767b 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -22,6 +22,21 @@ tokens stored anywhere. https://pypi.org/simple compileml` in a scratch venv, then `compileml --help` and a quick `import compileml`. +## Refreshing the benchmark + +Run it on an idle machine — the millisecond figures belong to the laptop's state +on the day — then let the script copy the numbers into the docs: + +```bash +python benchmarks/run_benchmarks.py +python benchmarks/sync_docs.py +``` + +Commit `benchmarks/results.json`, `benchmarks/selection_curve.json` and the +documents the script touched together. Retention figures are deterministic +and should not move between runs; if they do, something other than the +machine changed. + ## Releasing vX.Y.Z 1. Bump `__version__` in `src/compileml/__init__.py` — the single source of diff --git a/benchmarks/sync_docs.py b/benchmarks/sync_docs.py index ddc2fcc..e9c4a36 100644 --- a/benchmarks/sync_docs.py +++ b/benchmarks/sync_docs.py @@ -14,7 +14,6 @@ from __future__ import annotations -import io import json import re from pathlib import Path @@ -29,11 +28,13 @@ def read(path: Path) -> str: - return io.open(path, encoding="utf-8").read() + with open(path, encoding="utf-8") as f: + return f.read() def write(path: Path, text: str) -> None: - io.open(path, "w", encoding="utf-8", newline="\n").write(text) + with open(path, "w", encoding="utf-8", newline="\n") as f: + f.write(text) def replace_once(text: str, pattern: str, replacement: str, where: str, flags=re.M) -> str: @@ -42,18 +43,16 @@ def replace_once(text: str, pattern: str, replacement: str, where: str, flags=re raise SystemExit( f"{where}: expected exactly one match for {pattern!r}, found {len(matches)}" ) - return re.sub(pattern, replacement, text, count=1, flags=flags) + return re.sub(pattern, lambda _: replacement, text, count=1, flags=flags) def readme_table(res: dict) -> str: r, lat, art, det = res["retention"], res["latency"], res["artifact"], res["determinism"] lo, hi = r["retention_ci"] + retained = f"{r['gini_retention_pct']:.1f}% retained ({lo:.1f}–{hi:.1f})" rows = [ ("Ceiling Gini, 300-tree GBM", f"{r['teacher_gini']:.3f}"), - ( - "**Compiled integer artifact Gini**", - f"**{r['artifact_gini']:.3f} — {r['gini_retention_pct']:.1f}% retained ({lo:.1f}–{hi:.1f})**", - ), + ("**Compiled integer artifact Gini**", f"**{r['artifact_gini']:.3f} — {retained}**"), ( "Floor Gini, WoE logistic regression", f"{r['floor_gini']:.3f} — artifact at {r['floor_ratio_pct']:.1f}%",