diff --git a/benchmarking/automation/README.md b/benchmarking/automation/README.md index 0a5952d1be..089d0fbd79 100644 --- a/benchmarking/automation/README.md +++ b/benchmarking/automation/README.md @@ -165,13 +165,18 @@ gcloud projects add-iam-policy-binding \ ``` **Runner Job pod** (KSA `benchmark-runner` in namespace `benchmarking` on the -test cluster's project) needs `roles/storage.objectUser` on the destination -bucket so `runner.py` can upload results: +test cluster's project) needs `roles/storage.objectCreator` on the destination +bucket so `runner.py` can upload results, and `roles/storage.objectViewer` so +that a `cel` input with a `test` can find and read that test's run (such as +the suite's 1-user run) under `--dest` (without it, the variable is +`invalid` and says why): ```bash RUNNER_PRINCIPAL="principal://iam.googleapis.com/projects//locations/global/workloadIdentityPools/.svc.id.goog/subject/ns/benchmarking/sa/benchmark-runner" gcloud storage buckets add-iam-policy-binding gs:// \ --role=roles/storage.objectCreator --member="${RUNNER_PRINCIPAL}" +gcloud storage buckets add-iam-policy-binding gs:// \ + --role=roles/storage.objectViewer --member="${RUNNER_PRINCIPAL}" ``` ## Updating tests diff --git a/benchmarking/automation/orchestrator.py b/benchmarking/automation/orchestrator.py index 2bd6c6db3c..7700d808e9 100644 --- a/benchmarking/automation/orchestrator.py +++ b/benchmarking/automation/orchestrator.py @@ -406,6 +406,9 @@ def validate_and_normalize_tests(tests: list[dict[str, Any]]) -> None: if ttype == "locust": # A malformed --metadata fails here rather than mid-sweep. split_metadata_flag(t) + elif "evaluations" in t: + # Only the locust runner judges evaluations (runner.py --test-entry). + raise ValueError(f"test {name!r}: 'evaluations' is supported only for type locust") isolate_block(t) @@ -516,6 +519,9 @@ def run_test( if test_type(test) == "locust": flags, own = split_metadata_flag(test) flags += ["--metadata", json.dumps({**run_metadata(), **own})] + # The whole entry, so runner features (such as observers or evaluations) + # read what they need without adding a new flag per field. + flags += ["--test-entry", json.dumps(test)] manifest = render_template(tmpl, subs, flags) wait_for_no_active_runners() print(f"Submitting Job {job}", flush=True) diff --git a/benchmarking/automation/test_locust.py b/benchmarking/automation/test_locust.py index fec0aad397..c9f86f6b78 100644 --- a/benchmarking/automation/test_locust.py +++ b/benchmarking/automation/test_locust.py @@ -60,6 +60,13 @@ def test_missing_required(self): with self.assertRaises(ValueError): locust.validate(test) + # Shape only: the runner validates the rest (benchmarking/evaluate). + def test_evaluations_shape(self): + locust.validate(entry(evaluations=[{"name": "e", "anything": "else"}])) + for bad in ({"evaluations": {"name": "e"}}, {"evaluations": [{"name": ""}]}, {"evaluations": ["e"]}): + with self.subTest(bad=bad), self.assertRaisesRegex(ValueError, "locust test 't': evaluations must"): + locust.validate(entry(**bad)) + class PreTestTest(unittest.TestCase): def shaped(self, test): diff --git a/benchmarking/automation/test_orchestrator.py b/benchmarking/automation/test_orchestrator.py index b29084cd7a..fe0588c3fc 100644 --- a/benchmarking/automation/test_orchestrator.py +++ b/benchmarking/automation/test_orchestrator.py @@ -346,6 +346,26 @@ def test_bad_test_metadata_fails_clearly(self): with self.assertRaisesRegex(ValueError, "not valid JSON"): orchestrator.validate_and_normalize_tests([test]) + def test_locust_job_gets_the_whole_test_entry(self): + class Rendered(Exception): + pass + + fake = mock.Mock(job_tmpl=mock.Mock(return_value="t"), job_subs=mock.Mock(return_value={})) + test = {"name": "n", "type": "locust", "duration": "1m", "flags": ["--x"], "observers": [{"name": "pprof"}]} + with mock.patch.dict(orchestrator.TYPES, {"locust": fake}), \ + mock.patch.dict(os.environ, CLUSTER_ENV, clear=True), \ + mock.patch.object(orchestrator, "render_template", side_effect=Rendered) as render, \ + self.assertRaises(Rendered): + orchestrator.run_test(test, "img", "dest", "abcdef0123") + args = render.call_args.args[2] + self.assertEqual(json.loads(args[args.index("--test-entry") + 1]), test) + + def test_evaluations_only_on_locust(self): + test = {"name": "n", "type": "nighthawk-ingress", "targetCluster": "t", "evaluations": [{"name": "e"}]} + with mock.patch.object(orchestrator.TYPES["nighthawk-ingress"], "validate"), \ + self.assertRaisesRegex(ValueError, "test 'n': 'evaluations' is supported only for type locust"): + orchestrator.validate_and_normalize_tests([test]) + def test_metadata_survives_yaml(self): md = json.dumps({"cluster_name": "c: d", "q": "\"'"}) subs = {"JOB_NAME": "j", "IMAGE": "i", "TAG": "t", "NAME": "n", "DEST": "d"} diff --git a/benchmarking/automation/tests.yaml b/benchmarking/automation/tests.yaml index f1a239d81f..efe8e56d0d 100644 --- a/benchmarking/automation/tests.yaml +++ b/benchmarking/automation/tests.yaml @@ -112,6 +112,25 @@ # seconds, or the step falls short of its rate. A resumepaused iteration is a # resume plus a pause, so size it by the p99 of both together. # +# Optional `evaluations` judges the run against conditions on its results: +# runner.py writes the verdicts to evaluation.json and adds rows to +# stats.jsonl (evaluation_, and evaluations for the summary); they never +# fail the run. The orchestrator passes the whole entry to runner.py as +# --test-entry. A cel input's `test` names another test of the suite (such +# as its 1-user test), whose latest run under the same --dest with the same +# tag (at most 6h older, on the same cluster) it reads. See +# benchmarking/evaluate/README.md; check a file with +# `cd benchmarking && python3 -m evaluate --check-config --tests-file F`. +# +# evaluations: +# - name: envelope +# variables: +# - name: trial +# source: run_results +# source_cfg: {metric: trial_summary, measurements: [aggregate_failure_ratio]} +# conditions: +# - {name: failure_ratio, cel: "trial.aggregate_failure_ratio <= 0.001"} +# # An optional `controlPlane:` block shapes the control plane for a capacity # run; pre_test applies it after the deploy. Its `apiServer:` section, which # is required, sizes ate-api-server: diff --git a/benchmarking/automation/testtypes/locust.py b/benchmarking/automation/testtypes/locust.py index 3dcf542519..d490a0240c 100644 --- a/benchmarking/automation/testtypes/locust.py +++ b/benchmarking/automation/testtypes/locust.py @@ -20,6 +20,12 @@ An optional `controlPlane:` block shapes ate-api-server and Postgres for a capacity run (see controlplane.py); without it pre_test leaves the cluster as installed. + +The runner gets the whole entry (--test-entry) and judges the run against +its optional `evaluations:` (benchmarking/evaluate/README.md). This image +doesn't ship the evaluate package, so validate checks their shape only; the +runner checks the rest before its run, and `python -m evaluate +--check-config` checks a tests file offline. """ import os @@ -38,11 +44,22 @@ def control_plane_config(test: dict[str, Any]) -> dict[str, Any] | None: return controlplane.config(test["controlPlane"]) +def validate_evaluations_shape(test: dict[str, Any]) -> None: + """Raises ValueError unless evaluations is a list of mappings with a name.""" + name = test.get("name") + evaluations = test.get("evaluations", []) + if not isinstance(evaluations, list) or not all( + isinstance(e, dict) and isinstance(e.get("name"), str) and e["name"] for e in evaluations + ): + raise ValueError(f"locust test {name!r}: evaluations must be a list of mappings, each with a name") + + def validate(test: dict[str, Any]) -> None: name = test.get("name") for field in ("file", "duration", "users"): if field not in test: raise ValueError(f"locust test {name!r} missing {field!r}") + validate_evaluations_shape(test) if "controlPlane" not in test: return try: diff --git a/benchmarking/evaluate/README.md b/benchmarking/evaluate/README.md index 0a144e9b40..8b330f6ff2 100644 --- a/benchmarking/evaluate/README.md +++ b/benchmarking/evaluate/README.md @@ -9,7 +9,7 @@ Depends on [`cel-expr-python`](https://github.com/cel-expr/cel-python) and - A **source** reads the run and returns several named **values**, or says it can't judge the run (`invalid`, with a reason). Sources: - `run_results`. + `cel`, `run_results`. - A **variable** is one source query: `{name, source, source_cfg}`. Its values are exposed to CEL as a map under `` (`.`, or `[""]` when `` contains dots or CEL keywords). @@ -47,7 +47,8 @@ checks two envelope health gates: ``` - Evaluation, variable and condition names match `^[a-z][a-z0-9_]*$`, at most - 64 characters, and are unique where they live. CEL itself rejects a + 64 characters, and are unique where they live. A variable name can't contain + `__` or end in `_` (see [Outputs](#outputs)). CEL itself rejects a variable named after a CEL type (`int`, `map`) or a keyword (`in`, `true`) when a condition compiles. Value names come from the source and may be any non-empty string @@ -87,7 +88,7 @@ A source is a class with three hooks (`sources/base.py`): | hook | runs | does | |---|---|---| | `validate(cfg)`, a classmethod | when the evaluations are loaded | raises `ValueError` for a bad `source_cfg` | -| `init(run)` | once, when the run starts (its files may not exist yet) | takes the `BenchmarkRunContext`, the run being evaluated: which run it is (`identity`: `uri`, and `test_name`, `tag`, `run_date`, `run_ts` when known), where it is stored, and its stored artifacts (`artifact(name)`, a local path or None); the source chooses what it reads | +| `init(run)` | once, when the run starts (its files may not exist yet) | takes the `BenchmarkRunContext`, the run being evaluated: which run it is (`identity`: `uri`, and `test_name`, `tag`, `run_date`, `run_ts` when known), where it is stored, and its stored artifacts (`artifact(name)`, a local path or None); the source chooses what it reads. `run_of(test)` returns the run of another test from the same invocation or raises `LookupError` | | `query(cfg)` | after the run, once per distinct `source_cfg` | returns named values (numbers, bools, strings or null), or raises `SourceInvalid(reason, values)` when it can't judge the run | Value names come from `source_cfg` or are fixed by the source, never from the @@ -99,6 +100,200 @@ a value of another type, such as a list or a mapping, or a value name that isn't a non-empty string. Sources never write files. A new source is a class in `sources/` and a line in `SOURCES`. +### `cel` + +Binds data read from runs to CEL, and evaluates named expressions into +values. The source has no notion of what the data means: what counts as +good, which run to compare against, and every formula live in the config. + +```yaml +source: cel +source_cfg: + inputs: # the data, each a CEL variable + - name: stats + reader: json_lines + reader_cfg: {file: stats.jsonl} + locals: # optional: ordered list of named in-between results + - name: trials + cel: "stats.lines.filter(l, l.metric == 'trial_summary')" + checks: # optional: what must hold for the values to mean anything + - name: has_trial + cel: "size(trials) > 0" + values: # the outputs + - name: cores + cel: "size(trials) > 0 ? dyn(trials[size(trials) - 1].measurements.allocatable_cores) : null" +``` + +- `inputs: [{name: , from_test: ?, reader: , reader_cfg: {...}}, ...]`: + each input names a reader and its `reader_cfg`, and the reader's data + becomes the CEL variable ``. By default the reader reads this run. With `from_test:` it + reads that test's run from the same invocation (`run_of`), which lets a + config compare two runs. +- `locals: [{name: , cel: }, ...]`: ordered list of named in-between + results, such as a filtered list or a median. They are evaluated top to + bottom, so each local may read inputs and earlier locals by bare name. + Entries can share locals through YAML anchors. A local holds whatever CEL + produces, including `timestamp` and `duration`. +- `checks: [{name: , cel: }, ...]`: conditions the data must meet before the + values mean anything (for example, both runs used the same mode). A check + that is false or fails to evaluate makes the variable `invalid`, and the + reason names the check. Unlike an evaluation's conditions, a check judges + the data, not the run. +- `values: [{name: , cel: }, ...]`: the outputs. Each must give a double, bool, + string or `null`; a whole number such as `size(x)` must be converted with + `double()`. Use CEL's native `null` with `dyn()` for missing or empty cases + (`size(xs) > 0 ? dyn(...) : null`). A non-finite float (`0.0 / 0.0`) also + becomes `null`. +- Names match `^[a-z][a-z0-9_]*$` and are unique within each list. An input + and a local can't share a name. + Whole numbers stay integers here (`size(x) - 1` indexes a list), so write + doubles as `1500.0`. +- Functions beyond standard CEL: the CEL math extension (`ext_math`: + `math.least`, `math.greatest`, `math.round`, ...), plus two explicitly + defined numeric helpers: `round(x, digits)` (Python-style rounding, shared + with evaluation conditions) and `percentile(xs, p)` over `list(double)` + (linear interpolation between the closest ranks, `p` from 0 to 100, raising + an error on an empty list like `math.least`/`math.greatest`). + +What is checked, when, and what happens on a mismatch: + +| what | when | on a mismatch | +|---|---|---| +| unknown `source_cfg` key or reader, bad `reader_cfg`, bad or repeated names | config load | the evaluations fail to load | +| `select` / `until` expressions on `json_lines` | config load | load error | +| every expression compiles with inferred local types, and value return types can be double, bool, string or null | config load | load error | +| reader output has its declared `fields`, of their types | run time | `TypeError` makes the variable `invalid` | +| missing JSON keys inside `lines` or `data` | run time | reading a missing key makes the variable `invalid`, unless guarded with `has()` | +| a reader can't read its run (absent file, unhashable `unique_by`, `select` eval error, no run of `from_test`) | run time | `invalid`, and the reason names the input | +| a local, check or value fails to evaluate, or a value gives another type | run time | `invalid`, and the reason names it | + +A reader is a class in `sources/cel/` and a line in `READERS` +(`sources/cel/reader.py`): its `Config` (a pydantic model of its +`reader_cfg`), its `fields` (the top-level fields of its data and their +types, checked at run time; inputs are `map(string, dyn)` in CEL, so a field +name is not checked at config load) and `read(run, cfg)`, which raises +`ReadError` when the run can't be read. A reader exposes data and +passes no judgment on it. All numbers are doubles, and a field a record lacks +is absent rather than null. + +#### `json` + +One JSON object file from the run: `{name: summary, reader: json, reader_cfg: {file: server_summary.json}}`. + +- `data`: the parsed JSON object, with leaf coercion as in `run_results`. +- `uri`: where the run is stored. + +#### `json_lines` + +Filtered lines of one of the run's JSON-lines or prefixed-JSON files: + +```yaml +- name: records + reader: json_lines + reader_cfg: + file: logs.txt + prefix: "[boomer] " + until: "has(line.msg) && line.msg == 'sweperf shutdown started'" + select: "has(line.msg) && line.msg == 'sweperf task record' && has(line.v) && line.v == 1.0" +``` + +- `lines`: selected JSON objects, in file order. Values are coerced as in + `run_results`: numbers and numeric strings without `_` become doubles, + `"true"`/`"false"` become bools, and a null or non-finite number is left out. +- `unparsed_lines`: `[{line_no, text, error}]` for each prefixed (or, when + `prefix` is unset, non-blank) line that isn't a JSON object. +- `uri`: where the run is stored. +- Optional `reader_cfg` keys: + - `prefix`: skip lines not starting with `prefix` and strip it before + parsing JSON (without it, blank lines are skipped). + - `until`: CEL boolean expression over `line`; stop reading when a parsed + object matches (that line is not emitted). + - `select`: CEL boolean expression over `line`; keep only matching objects. + - `unique_by`: field name; keep only the first selected object per distinct + value of that field (useful to bound list size before CEL evaluation). + +Goodput, as config only: a task is good when it ended in the steady phase, +didn't fail, woke within limits (slowest resume 1.5 s, slowest time to first +execute 1 s), and its command time (`cel_ms`) and wall time stayed within 1.5x +of the 1-user run's medians. + +```yaml +- name: sweperf_astropy_150_users_think + # ... + evaluations: + - name: goodput_strict + variables: + - name: goodput + source: cel + source_cfg: + inputs: + - name: records + reader: json_lines + reader_cfg: + file: logs.txt + prefix: "[boomer] " + until: "has(line.msg) && line.msg == 'sweperf shutdown started'" + select: "has(line.msg) && line.msg == 'sweperf task record' && has(line.v) && line.v == 1.0" + - name: failures + reader: json_lines + reader_cfg: + file: logs.txt + prefix: "[boomer] " + until: "has(line.msg) && line.msg == 'sweperf shutdown started'" + select: "has(line.msg) && line.msg in ['ResumeActor failed in step', 'execute steps failed', 'SuspendActor failed', 'SuspendActorAsync failed'] && has(line.actor) && line.actor != ''" + unique_by: actor + - name: shutdown + reader: json_lines + reader_cfg: + file: logs.txt + prefix: "[boomer] " + select: "has(line.msg) && line.msg == 'sweperf shutdown started'" + - {name: summary, reader: json, reader_cfg: {file: server_summary.json}} + - {name: stats, reader: json_lines, reader_cfg: {file: stats.jsonl}} + - name: one_user + from_test: sweperf_astropy_1_user + reader: json_lines + reader_cfg: + file: logs.txt + prefix: "[boomer] " + until: "has(line.msg) && line.msg == 'sweperf shutdown started'" + select: "has(line.msg) && line.msg == 'sweperf task record' && has(line.v) && line.v == 1.0" + - {name: one_user_stats, from_test: sweperf_astropy_1_user, reader: json_lines, reader_cfg: {file: stats.jsonl}} + locals: + - {name: lo, cel: "timestamp(int(summary.data.metadata.steady_start_ts))"} + - {name: steady_end, cel: "timestamp(int(summary.data.metadata.steady_end_ts))"} + - {name: stop, cel: "size(shutdown.lines) > 0 ? timestamp(shutdown.lines[0].time) : steady_end"} + - {name: hi, cel: "stop < steady_end ? stop : steady_end"} + - {name: window_s_raw, cel: "double((hi - lo).getSeconds()) + double((hi - lo).getMilliseconds()) / 1000.0"} + - {name: hours, cel: "math.greatest(window_s_raw, 0.0) / 3600.0"} + - {name: done, cel: "records.lines.filter(t, timestamp(t.end) >= lo && timestamp(t.end) <= hi)"} + - {name: recorded, cel: "records.lines.map(t, t.actor)"} + - {name: lost, cel: "failures.lines.filter(f, !(f.actor in recorded) && timestamp(f.time) >= lo && timestamp(f.time) <= hi)"} + - {name: ref_cel_ms, cel: "round(percentile(one_user.lines.filter(t, !t.failed).map(t, t.cel_ms), 50.0), 3)"} + - {name: ref_wall_ms, cel: "round(percentile(one_user.lines.filter(t, !t.failed).map(t, t.wall_ms), 50.0), 3)"} + - {name: trials, cel: "stats.lines.filter(l, l.metric == 'trial_summary')"} + - {name: cores, cel: "trials[size(trials) - 1].measurements.allocatable_cores"} + - {name: run_async, cel: "stats.lines.exists(l, has(l.metric) && l.metric == 'grpc_SuspendActorAsync')"} + - {name: run_sync, cel: "stats.lines.exists(l, has(l.metric) && l.metric == 'grpc_SuspendActor')"} + - {name: one_async, cel: "one_user_stats.lines.exists(l, has(l.metric) && l.metric == 'grpc_SuspendActorAsync')"} + - {name: one_sync, cel: "one_user_stats.lines.exists(l, has(l.metric) && l.metric == 'grpc_SuspendActor')"} + - {name: run_mode, cel: "run_async && run_sync ? 'mixed' : (run_async ? 'async' : (run_sync ? 'sync' : ''))"} + - {name: one_mode, cel: "one_async && one_sync ? 'mixed' : (one_async ? 'async' : (one_sync ? 'sync' : ''))"} + - {name: good, cel: "done.filter(t, !t.failed && t.resume_max_ms <= 1500.0 && t.ttfe_max_ms <= 1000.0 && t.cel_ms <= 1.5 * ref_cel_ms && t.wall_ms <= 1.5 * ref_wall_ms)"} + checks: + - {name: has_shutdown, cel: "size(shutdown.lines) > 0"} + - {name: same_lifecycle_mode, cel: "'' in [run_mode, one_mode] || run_mode == one_mode"} + - {name: one_user_has_clean_tasks, cel: "one_user.lines.exists(t, !t.failed)"} + - {name: tasks_in_steady_window, cel: "size(done) + size(lost) > 0"} + values: + - {name: good, cel: "double(size(good))"} + - {name: good_ratio_pct, cel: "round(100.0 * double(size(good)) / double(size(done) + size(lost)), 2)"} + - {name: good_per_vcpu_hour, cel: "round(double(size(good)) / hours / cores, 4)"} + - {name: cel_ms_ratio_p90, cel: "size(done) > 0 ? dyn(round(percentile(done.map(t, t.cel_ms / ref_cel_ms), 90.0), 3)) : null"} + conditions: + - {name: good_ratio, cel: "goodput.good_ratio_pct >= 80.0"} +``` + ### `run_results` The measurements of one `stats.jsonl` row: @@ -124,14 +319,68 @@ variable with its values (and why it is invalid), and the evaluation as loaded (`config`). An evaluation's reason names the conditions that failed, else the invalid conditions and variables. +## Outputs + +- `evaluation.json`: the `Report` as JSON. When the evaluations failed to + load, `error` says why and nothing is judged. +- `stats.jsonl` rows in the file's usual five keys: one `evaluation_` row + per evaluation and one `evaluations` row for the summary. Measurements are + strings, or null. An evaluation's row has its `status`, `passed`, `reason`, + `failed_conditions` and `invalid_conditions`, and each value as + `__`. Value names follow the rule in + [Sources](#sources), so the keys stay bounded. A variable name can't contain + `__` or end in `_`, so no two values share a key. + +## In the runner + +The orchestrator passes the whole `locust` test entry to `runner.py` as +`--test-entry` (JSON), and the runner reads its `evaluations`. Before the +load test, the runner loads them and runs the +sources' `init`, so a config error is in `logs.txt` from the start. As its +last step after the run, it runs the queries, writes `evaluation.json` (a +config error goes there too), uploads it with the run's other files, appends +the rows to `stats.jsonl`, and logs one line per evaluation. `status.json` and +the exit code don't change: an evaluation never fails a run. + +The runner's `run_of(test)` returns the run of `test` under the same +`--dest` with the same `--tag` and the latest `run_ts`, at most 6 h before +this run (`rundir.py`). When both runs record a cluster (`trial_summary`'s +`metadata_cluster_name`), it must be the same. On a `gs://` `--dest`, this +needs read access to the bucket (see the +[automation README](../automation/README.md#iam-prerequisites)). + +## Evaluating stored runs + +```sh +cd benchmarking +python3 -m evaluate RUN... (--evaluations JSON|@FILE | --tests-file FILE --test-name NAME | --from-run) [--out DIR] +python3 -m evaluate --check-config --tests-file automation/tests.yaml # check the blocks only +``` + +`RUN` is a run directory or `gs://.../runs//run_date=.../run_ts=.../run_tag=.../`, +whose artifacts are each downloaded to a temporary directory when a source +first reads one (through `gcloud storage`, or the google-cloud-storage library +when installed). A `RUN` stored in the runner's +layout, local or `gs://`, has the paired runs the runner would give it, under +the same ``, and any other `RUN` has none. `--from-run` re-reads the +evaluations the run was judged with, from its `evaluation.json`. The outputs +go to `--out`, by default `./evaluate-out//run_ts=/`, never into the +stored run: `evaluation.json` and `evaluation_rows.jsonl`. Exit status: 0 +evaluated (whatever the verdict), 2 config error, 1 I/O error. `--tests-file` +needs PyYAML. + ## Layout | file | | |---|---| | `config.py` | loads and validates an `evaluations` block (pydantic models) | -| `cel_env.py` | shared CEL environment, `round()`, and evaluation | +| `cel_env.py` | shared CEL environment, helpers (`round`, `percentile`), and evaluation | | `evaluator.py` | runs the source hooks and judges a run into a `Report` | +| `outputs.py` | `evaluation.json` and the `stats.jsonl` rows | | `sources/` | the hooks (`base.py`) and the sources | +| `rundir.py` | the runner's storage layout: opening local or `gs://` runs (a `gs://` artifact is downloaded on first use), finding a paired run | +| `__init__.py` | `prepare` and `write`, which `runner.py` calls | +| `__main__.py` | the CLI | ## Tests diff --git a/benchmarking/evaluate/__init__.py b/benchmarking/evaluate/__init__.py index 69ef3be208..ca6cdaa4b1 100644 --- a/benchmarking/evaluate/__init__.py +++ b/benchmarking/evaluate/__init__.py @@ -16,6 +16,40 @@ A test entry's `evaluations:` (config.py) name sources, which read the run's files into values (sources/), and CEL conditions over those values -(cel_env.py). evaluator.py judges them into a Report. Depends on -cel-expr-python and pydantic (requirements.txt). See README.md. +(cel_env.py). evaluator.py judges them into a Report, which outputs.py writes +as evaluation.json and stats.jsonl rows. runner.py calls prepare when a run +starts and write once it has finished. rundir.py finds the runs a run is +paired with under the runner's --dest. `python -m evaluate` (__main__.py) +judges a stored run with the same code. Depends on cel-expr-python +and pydantic (requirements.txt). See README.md. """ + +from pathlib import Path +from typing import Any, Optional + +from . import config, outputs +from .evaluator import Evaluator, Report, error_report # noqa: F401 +from .outputs import summary_lines # noqa: F401 +from .sources.base import BenchmarkRunContext, LocalArtifacts, RunIdentity # noqa: F401 + + +def check_config(evaluations: Any) -> Optional[str]: + """Why evaluations (a test entry's block, as parsed) is not valid, as "config: ..."; None when it is. Never raises.""" + try: + config.load(evaluations) + except ValueError as e: # a ConfigError + return f"config: {e}" + except Exception as e: # pylint: disable=broad-except # a shape no check expected + return f"config: {type(e).__name__}: {e}" + return None + + +def prepare(evaluations: Any, run: BenchmarkRunContext) -> Evaluator: + """The evaluator of run, whose sources have run their init hook. Raises when evaluations is invalid.""" + return Evaluator(config.load(evaluations), run) + + +def write(report: Report, out_json: Path, jsonl_path: Path, timestamp: str, tag: str, test_name: str) -> None: + """Writes report to out_json and appends its stats rows to jsonl_path.""" + outputs.write_evaluation_json(report, out_json) + outputs.append_jsonl(outputs.stats_rows(report, timestamp, tag, test_name), jsonl_path) diff --git a/benchmarking/evaluate/__main__.py b/benchmarking/evaluate/__main__.py new file mode 100644 index 0000000000..78162dd78f --- /dev/null +++ b/benchmarking/evaluate/__main__.py @@ -0,0 +1,189 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Evaluates stored benchmark runs with the code the runner uses, read-only on the runs. + + python -m evaluate RUN... (--evaluations JSON|@FILE | --tests-file FILE --test-name NAME | --from-run) + [--out DIR] + python -m evaluate --check-config (--evaluations JSON|@FILE | --tests-file FILE [--test-name NAME]) + +RUN is a run directory or gs://.../runs//run_date=.../run_ts=.../run_tag=.../ +(an artifact is downloaded when first read). A RUN stored in the runner's layout has the +paired runs the runner would give it, under the same ; any other RUN has +none. --from-run re-reads the evaluations a run was evaluated with (its +evaluation.json); give new ones to re-evaluate it with new conditions. Writes +evaluation.json and evaluation_rows.jsonl (the rows the runner appends to +stats.jsonl) to --out, by default ./evaluate-out//run_ts=/; never +into the stored run. + +Exit status: 0 evaluated (whatever the verdict), 2 config error, 1 I/O error. +""" + +import argparse +import json +import os +import sys +import tempfile +from dataclasses import replace +from pathlib import Path +from typing import Any, Optional + +from . import config, rundir, summary_lines, write +from .evaluator import Evaluator +from .sources.base import BenchmarkRunContext, RunIdentity + + +class UsageError(Exception): + pass + + +def read_json_arg(value: str) -> Any: + """JSON given inline or as @FILE.""" + text = Path(value[1:]).read_text(encoding="utf-8") if value.startswith("@") else value + return json.loads(text) + + +def tests_file_entries(path: str) -> list[dict[str, Any]]: + try: + import yaml # pylint: disable=import-outside-toplevel + except ImportError as e: + raise UsageError("--tests-file needs PyYAML (pip install pyyaml)") from e + with open(path, encoding="utf-8") as f: + doc = yaml.safe_load(f) or {} + tests = doc.get("tests") if isinstance(doc, dict) else None + if not isinstance(tests, list): + raise config.ConfigError(f"{path}: no top-level tests list") + return [t for t in tests if isinstance(t, dict)] + + +def tests_file_entry(path: str, name: str) -> dict[str, Any]: + for t in tests_file_entries(path): + if t.get("name") == name: + return t + raise config.ConfigError(f"{path}: no test named {name!r}") + + +def open_run(uri: str, tmp: Path) -> BenchmarkRunContext: + """The run at uri, with the runner's paired runs when it is stored in the runner's layout.""" + # Identity comes from the path as given, unresolved: a link placed in a runs/ tree is named by its own path. + run = rundir.open_run(uri if uri.startswith("gs://") else os.path.abspath(uri), tmp / "run") + dest = rundir.dest_of(run.identity.uri) + if dest is not None: + run_of = rundir.paired_run_finder(dest, run.identity.tag, run.identity.run_ts, run.artifacts, + tmp / "paired") + run = replace(run, run_of=run_of) + return run + + +def from_run_evaluations(run: BenchmarkRunContext, uri: str) -> Any: + path = run.artifact("evaluation.json") + if path is None: + raise config.ConfigError(f"--from-run: {uri} has no evaluation.json") + report = json.loads(path.read_text(encoding="utf-8")) + if report.get("error"): + raise config.ConfigError(f"--from-run: the run's evaluations did not load: {report['error']}") + return [ev["config"] for ev in report.get("evaluations", [])] + + +def check_config(a: argparse.Namespace) -> int: + if a.evaluations is not None: + blocks = [("--evaluations", read_json_arg(a.evaluations))] + elif a.tests_file and a.test_name: + blocks = [(a.test_name, tests_file_entry(a.tests_file, a.test_name).get("evaluations", []))] + elif a.tests_file: + blocks = [(t.get("name"), t["evaluations"]) for t in tests_file_entries(a.tests_file) if "evaluations" in t] + else: + raise UsageError("--check-config needs --evaluations or --tests-file") + bad = 0 + for name, block in blocks: + try: + n = len(config.load(block)) + print(f"{name}: ok ({n} evaluations)") + except config.ConfigError as e: + bad += 1 + print(f"{name}: {e}", file=sys.stderr) + except Exception as e: # pylint: disable=broad-except # a shape no check expected + bad += 1 + print(f"{name}: {type(e).__name__}: {e}", file=sys.stderr) + return 2 if bad else 0 + + +def out_dir(a: argparse.Namespace, identity: RunIdentity, several: bool) -> Path: + run_ts = "unknown" if identity.run_ts is None else identity.run_ts + leaf = Path(identity.test_name or Path(identity.uri).name) / f"run_ts={run_ts}" + if a.out and not several: + return Path(a.out) + return Path(a.out or "evaluate-out") / leaf + + +def evaluate_runs(a: argparse.Namespace) -> int: + with tempfile.TemporaryDirectory() as tmp: + raw_evaluations = None + if a.evaluations is not None: + raw_evaluations = read_json_arg(a.evaluations) + elif a.tests_file: + if not a.test_name: + raise UsageError("--tests-file needs --test-name") + raw_evaluations = tests_file_entry(a.tests_file, a.test_name).get("evaluations", []) + elif not a.from_run: + raise UsageError("give --evaluations, --tests-file with --test-name, or --from-run") + evaluations = config.load(raw_evaluations) if raw_evaluations is not None else None + for i, uri in enumerate(a.runs): + run = open_run(uri, Path(tmp) / f"r{i}") + evals = evaluations if evaluations is not None else config.load(from_run_evaluations(run, uri)) + report = Evaluator(evals, run).evaluate() + out = out_dir(a, run.identity, len(a.runs) > 1) + # The rows carry the run's own data timestamp, as the runner's do. + stamp = rundir.data_timestamp(run.artifacts) or report.evaluated_at + (out / "evaluation_rows.jsonl").unlink(missing_ok=True) + write(report, out / "evaluation.json", out / "evaluation_rows.jsonl", stamp, run.identity.tag or "", + run.identity.test_name or "") + print(f"== {uri}") + for line in summary_lines(report): + print(line) + print(f"wrote {out / 'evaluation.json'}", flush=True) + return 0 + + +def main(argv: Optional[list[str]] = None) -> int: + p = argparse.ArgumentParser(prog="python -m evaluate", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("runs", nargs="*", metavar="RUN", help="run directory or gs:// run path") + src = p.add_mutually_exclusive_group() + src.add_argument("--evaluations", help="the evaluations block as JSON, or @FILE") + src.add_argument("--tests-file", help="a tests.yaml; with --test-name, that entry's evaluations") + src.add_argument("--from-run", action="store_true", help="the evaluations in the run's evaluation.json") + p.add_argument("--test-name", help="the tests-file entry") + p.add_argument("--out", help="output directory (default ./evaluate-out//run_ts=/)") + p.add_argument("--check-config", action="store_true", help="validate the evaluations only") + a = p.parse_args(argv) + try: + if a.check_config: + return check_config(a) + if not a.runs: + raise UsageError("give at least one RUN") + return evaluate_runs(a) + except UsageError as e: + p.error(str(e)) + except (config.ConfigError, json.JSONDecodeError) as e: + print(f"config error: {e}", file=sys.stderr) + return 2 + except OSError as e: + print(f"I/O error: {e}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarking/evaluate/cel_env.py b/benchmarking/evaluate/cel_env.py index 532d667437..b1aef384fc 100644 --- a/benchmarking/evaluate/cel_env.py +++ b/benchmarking/evaluate/cel_env.py @@ -12,9 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Shared CEL environment, round(), and evaluation.""" +"""Shared CEL environment, explicit helpers, and evaluation. -from typing import Any, Mapping +Beyond standard CEL and its math extension (ext_math), only two explicit +numeric helpers are exposed: +- round(x, digits): Python-style rounding for doubles (shared with evaluation conditions). +- percentile(xs, p): linear-interpolation percentile over list(double), raising + ValueError on an empty list like math.least/math.greatest. +""" + +import math +from typing import Any, Mapping, Optional from cel_expr_python import cel @@ -31,6 +39,29 @@ def _round(x: float, digits: float) -> float: impl=_round) for t in (cel.Type.DOUBLE, cel.Type.INT)]) +def _percentile(xs: list, p: float) -> float: + """The p-th percentile of xs (p from 0 to 100), interpolated linearly + between the closest ranks; raises ValueError when xs is empty.""" + if not 0.0 <= p <= 100.0: + raise ValueError(f"percentile: p {p} isn't between 0 and 100") + s = sorted(float(x) for x in xs) + if not s: + raise ValueError("percentile: empty list") + if not all(math.isfinite(x) for x in s): + raise ValueError("percentile: non-finite value") + k = (len(s) - 1) * p / 100.0 + lo = math.floor(k) + hi = min(lo + 1, len(s) - 1) + return s[lo] + (s[hi] - s[lo]) * (k - lo) + + +PERCENTILE = cel.FunctionDecl( + "percentile", + [cel.Overload("percentile_list_double", return_type=cel.Type.DOUBLE, + parameters=[cel.Type.List(cel.Type.DOUBLE), cel.Type.DOUBLE], impl=_percentile)], +) + + def to_cel(values: Mapping[str, Any]) -> dict[str, Any]: """values as CEL reads them: ints as doubles, and no key for None, so has() tests presence. Source values come from JSON, which has one number type, and @@ -57,21 +88,27 @@ def evaluate_condition(expr: cel.Expression, data: Mapping[str, Any]) -> bool: class Env: """A CEL environment with variables and functions.""" - def __init__(self, variables: Mapping[str, cel.Type], functions: tuple[cel.FunctionDecl, ...] = ()): + def __init__( + self, + variables: Mapping[str, cel.Type], + functions: tuple[cel.FunctionDecl, ...] = (), + extensions: tuple[cel.CelExtensionBase, ...] = (), + ): self.variables = dict(variables) try: - self.env = cel.NewEnv(variables=self.variables, functions=list(functions)) + self.env = cel.NewEnv(variables=self.variables, functions=list(functions), extensions=list(extensions)) except RuntimeError as e: raise ValueError(str(e)) from None def compile(self, expression: str, where: str, - gives: tuple[cel.Type, ...] = (cel.Type.BOOL,)) -> cel.Expression: - """expression compiled. Raises ValueError unless it type-checks and gives one of gives.""" + gives: Optional[tuple[cel.Type, ...]] = (cel.Type.BOOL,)) -> cel.Expression: + """expression compiled. Raises ValueError unless it type-checks and gives + one of gives (any type when gives is None).""" try: expr = self.env.compile(expression) except RuntimeError as e: # a syntax or type error, or a variable CEL already declares raise ValueError(f"{where}: {e}") from None - if expr.return_type() not in gives: + if gives is not None and expr.return_type() not in gives: wanted = " or a ".join(t.name().lower() for t in gives if t != cel.Type.DYN) raise ValueError(f"{where}: must give a {wanted}, gives {expr.return_type().name()}") return expr diff --git a/benchmarking/evaluate/config.py b/benchmarking/evaluate/config.py index c7cb11e155..f59cae0345 100644 --- a/benchmarking/evaluate/config.py +++ b/benchmarking/evaluate/config.py @@ -44,8 +44,13 @@ class ConfigError(ValueError): """The evaluations block is invalid; the message names where.""" +# A variable name can't contain "__" or end in "_": stats.jsonl rows key each value as +# __, so trial__x with value y and trial with value x__y would collide. +VariableName = Annotated[str, Field(pattern=r"^[a-z][a-z0-9]*(_[a-z0-9]+)*$", max_length=64)] + + class Variable(Model): - name: Name + name: VariableName source: str # JSON, as the results echo it: a YAML date or a non-string key is an error here. # Validation builds new containers, so later edits to the input don't reach it. diff --git a/benchmarking/evaluate/evaluator.py b/benchmarking/evaluate/evaluator.py index 05d7652494..d39f6f257c 100644 --- a/benchmarking/evaluate/evaluator.py +++ b/benchmarking/evaluate/evaluator.py @@ -35,7 +35,7 @@ from . import cel_env from .config import Condition, Evaluation, to_json from .sources import SOURCES -from .sources.base import BenchmarkRunContext, Scalar, Source, SourceInvalid +from .sources.base import BenchmarkRunContext, RunIdentity, Scalar, Source, SourceInvalid Status = Literal["pass", "fail", "invalid"] PASS: Final = "pass" @@ -85,7 +85,8 @@ class Summary: class Report: """A run judged against its evaluations.""" run: Mapping[str, str] # which run was judged: its RunIdentity.record() - evaluated_at: str + evaluated_at: Optional[str] # None when the evaluations failed to load + error: Optional[str] # why the evaluations failed to load, None when they loaded summary: Summary evaluations: tuple[EvaluationResult, ...] @@ -143,7 +144,8 @@ def evaluate(self, now: Optional[datetime] = None) -> Report: name=ev.name, description=ev.description, status=status, passed=status == PASS, reason=reason, conditions=conditions, variables=tuple(variables), config=echoed)) status, failed, invalid = _combine(evaluation_results) - return Report(run=self.run.identity.record(), evaluated_at=now.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + return Report(run=self.run.identity.record(), + evaluated_at=now.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), error=None, summary=Summary(status=status, passed=status == PASS, failed=failed, invalid=invalid), evaluations=tuple(evaluation_results)) @@ -170,6 +172,12 @@ def _resolve_variable(self, source_name: str, cfg: Mapping[str, Any]) -> Resolve return ResolvedValues(values, invalid_reason) +def error_report(identity: RunIdentity, error: str) -> Report: + """The Report of evaluations that failed to load: no verdict, and why.""" + return Report(run=identity.record(), evaluated_at=None, error=error, + summary=Summary(status=INVALID, passed=False, failed=(), invalid=()), evaluations=()) + + def _combine(results: Sequence[ConditionResult | EvaluationResult]) -> tuple[Status, tuple[str, ...], tuple[str, ...]]: """fail if any result failed, else invalid if any is invalid, else pass; and the names of each.""" failed = tuple(r.name for r in results if r.status == FAIL) diff --git a/benchmarking/evaluate/outputs.py b/benchmarking/evaluate/outputs.py new file mode 100644 index 0000000000..79510762cf --- /dev/null +++ b/benchmarking/evaluate/outputs.py @@ -0,0 +1,104 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The outputs of an evaluation: evaluation.json, and summary rows for stats.jsonl. + +evaluation.json holds the Report as JSON: per evaluation its status, each +condition with its CEL expression and verdict, each variable with all its +values, and the evaluation as loaded. The stats.jsonl rows reuse the file's +five keys (timestamp, tag, test_name, metric, measurements) so the existing +pipeline picks them up: one `evaluation_` row per evaluation and one +`evaluations` row for the summary, with string measurements like the others. +""" + +import dataclasses +import json +from pathlib import Path +from typing import Any, Optional + +from .evaluator import FAIL, INVALID, Report + +# The summary row's metric, which no evaluation_ can equal. +SUMMARY_METRIC = "evaluations" + + +def write_evaluation_json(report: Report, path: Path) -> None: + text = json.dumps(dataclasses.asdict(report), indent=2, allow_nan=False) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text + "\n", encoding="utf-8") + + +def _str(v: Any) -> Optional[str]: + if v is None: + return None + if isinstance(v, bool): + return "true" if v else "false" + if isinstance(v, float) and v.is_integer(): + return str(int(v)) if abs(v) < 1e15 else repr(v) + return str(v) + + +def stats_rows(report: Report, timestamp: str, tag: str, test_name: str) -> list[dict[str, Any]]: + """stats.jsonl rows for report: one per evaluation, then the summary. + + An evaluation's row has its status, passed, reason, failed_conditions and + invalid_conditions, and each value of each variable as __, so a dashboard reads the numbers next to the verdict. Variable names + come from the config, and value names follow the query hook's rule + (sources/base.py), so the keys stay bounded. A variable name has no "__" + and doesn't end in "_" (config.py), so no two values share a key. A report + whose evaluations failed to load gets only the summary row, carrying the + error. + """ + def row(metric: str, measurements: dict[str, Any]) -> dict[str, Any]: + return {"timestamp": timestamp, "tag": tag, "test_name": test_name, "metric": metric, + "measurements": {k: _str(v) for k, v in measurements.items()}} + + rows = [] + for ev in report.evaluations: + m: dict[str, Any] = { + "status": ev.status, "passed": ev.passed, "reason": ev.reason, + "failed_conditions": ",".join(c.name for c in ev.conditions if c.status == FAIL), + "invalid_conditions": ",".join(c.name for c in ev.conditions if c.status == INVALID), + } + for var in ev.variables: + for name, value in var.values.items(): + m[f"{var.name}__{name}"] = value + rows.append(row(f"evaluation_{ev.name}", m)) + s = report.summary + rows.append(row(SUMMARY_METRIC, {"status": s.status, "passed": s.passed, "failed": ",".join(s.failed), + "invalid": ",".join(s.invalid), "error": report.error or ""})) + return rows + + +def append_jsonl(rows: list[dict[str, Any]], path: Path) -> None: + lines = "".join(json.dumps(r) + "\n" for r in rows) + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "a", encoding="utf-8") as f: + f.write(lines) + + +def summary_lines(report: Report) -> list[str]: + """One line per evaluation, with failed or invalid condition reasons, for logs.""" + if report.error: + return [f"evaluations not run: {report.error}"] + lines = [] + for ev in report.evaluations: + line = f"evaluation {ev.name}: {ev.status}" + if ev.reason: + line += f" ({ev.reason})" + details = list(dict.fromkeys(c.reason for c in ev.conditions if c.reason)) + lines.append(line + "".join(f"\n {d}" for d in details)) + lines.append(f"summary: {report.summary.status} ({len(report.evaluations)} evaluations)") + return lines diff --git a/benchmarking/evaluate/requirements.txt b/benchmarking/evaluate/requirements.txt index ebd31f6ffe..1aa84b8160 100644 --- a/benchmarking/evaluate/requirements.txt +++ b/benchmarking/evaluate/requirements.txt @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# cel-expr-python for CEL conditions, pydantic for the config's shape. +# cel-expr-python for CEL conditions and the cel source, pydantic for the config's shape. +# PyYAML is for the tests and for `python -m evaluate --tests-file`. cel-expr-python==0.1.3 pydantic>=2 +PyYAML diff --git a/benchmarking/evaluate/rundir.py b/benchmarking/evaluate/rundir.py new file mode 100644 index 0000000000..e238aedb22 --- /dev/null +++ b/benchmarking/evaluate/rundir.py @@ -0,0 +1,227 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Where runs are stored: opening stored runs, and finding the run another run is paired with. + +The runner stores each run under +/runs//run_date=/run_ts=/run_tag=/, with + a local directory or gs://bucket/path. This module is the only one that +knows that layout or touches gs://: it opens a stored run as a +BenchmarkRunContext whose identity comes from the layout and whose artifacts +are read from the store, a gs:// one downloaded on first use (read-only on the +store). GCS is read through the google-cloud-storage client when it is +installed (the runner image), else through `gcloud storage`. +""" + +import re +import shutil +import subprocess +import tempfile +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Callable, Optional + +from .sources.base import ArtifactStore, BenchmarkRunContext, LocalArtifacts, RunIdentity +from .sources.run_results import read_stats + +RUN_PATH = re.compile(r"^(?P.*)/runs/(?P[^/]+)/run_date=(?P[^/]+)/run_ts=(?P\d+)" + r"/run_tag=(?P[^/]+)/?$") +DEFAULT_MAX_AGE_S = 6 * 3600 + + +def identity_from_uri(uri: str) -> RunIdentity: + """The identity of the run stored at uri: test_name, tag, run_date and run_ts too when it is in the runner's layout.""" + uri = uri.rstrip("/") + m = RUN_PATH.match(uri) + if not m: + return RunIdentity(uri=uri) + return RunIdentity(uri=uri, test_name=m["test"], tag=m["tag"], run_date=m["date"], run_ts=int(m["ts"])) + + +def run_uri(dest: str, test: str, run_date: str, run_ts: str, tag: str) -> str: + """The uri of a run stored in the runner's layout; RUN_PATH parses it.""" + return f"{dest.rstrip('/')}/runs/{test}/run_date={run_date}/run_ts={run_ts}/run_tag={tag}" + + +class GcsArtifacts: + """A run's artifacts under a gs:// prefix, each downloaded on first use into tmp. Never writes to GCS.""" + + def __init__(self, uri: str, tmp: Path): + self.uri = uri.rstrip("/") + self.tmp = tmp + self._fetched: dict[str, Optional[Path]] = {} + + def fetch(self, name: str) -> Optional[Path]: + """Raises OSError on a read error.""" + if name not in self._fetched: + self._fetched[name] = _gcs_download(f"{self.uri}/{name}", self.tmp / name) + return self._fetched[name] + + +def open_run(uri: str, tmp: Path) -> BenchmarkRunContext: + """A BenchmarkRunContext for a local run directory, or a gs:// run whose artifacts are downloaded under tmp.""" + uri = uri.rstrip("/") + if not uri.startswith("gs://"): + if not Path(uri).is_dir(): + raise FileNotFoundError(f"no run directory {uri}") + return BenchmarkRunContext(identity_from_uri(uri), LocalArtifacts(Path(uri))) + if not _gcs_list(f"{uri}/"): + raise FileNotFoundError(f"no objects under {uri}/") + tmp.mkdir(parents=True, exist_ok=True) + local = Path(tempfile.mkdtemp(prefix="run-", dir=tmp)) + return BenchmarkRunContext(identity_from_uri(uri), GcsArtifacts(uri, local)) + + +def dest_of(uri: str) -> Optional[str]: + """The a run stored in the runner's layout lives under; None otherwise.""" + m = RUN_PATH.match(uri.rstrip("/")) + return m["root"] if m else None + + +def find_paired_run(dest: str, test_name: str, tag: str, before_ts: int, max_age_s: int = DEFAULT_MAX_AGE_S) -> str: + """The stored run of test_name paired with a run of tag at before_ts; raises LookupError when there is none. + + The pair is the run of test_name under dest with the same tag (commit) and + the latest run_ts at or before before_ts, at most max_age_s earlier: in a + suite, the earlier test of the same invocation. It must have a logs.txt or + a stats.jsonl. + """ + first = datetime.fromtimestamp(before_ts - max_age_s, tz=timezone.utc).date() + last = datetime.fromtimestamp(before_ts, tz=timezone.utc).date() + dates = [(first + timedelta(days=i)).isoformat() for i in range((last - first).days + 1)] + candidates = [] + for date in dates: + for found in _list_runs(f"{dest.rstrip('/')}/runs/{test_name}/run_date={date}/"): + other = identity_from_uri(found) + if other.tag == tag and other.run_ts is not None and before_ts - max_age_s <= other.run_ts <= before_ts: + candidates.append((other.run_ts, found)) + if not candidates: + raise LookupError(f"no run of {test_name} with tag {tag} under {dest.rstrip('/')}/runs/ in the " + f"{max_age_s / 3600:g} h before run_ts {before_ts}") + return max(candidates)[1] + + +def cluster_name(artifacts: ArtifactStore) -> str: + """The cluster a run ran on (trial_summary's metadata_cluster_name); "" when not recorded. + Raises ValueError when stats.jsonl is corrupt, OSError when it can't be read.""" + trial = read_stats(artifacts.fetch("stats.jsonl")).get("trial_summary") or {} + measurements = trial.get("measurements") + name = measurements.get("metadata_cluster_name") if isinstance(measurements, dict) else None + return str(name) if name else "" + + +def data_timestamp(artifacts: ArtifactStore) -> str: + """The timestamp the run's stats.jsonl rows carry (trial_summary's); "" without one + or when stats.jsonl is corrupt (the evaluation then says why).""" + try: + stats = read_stats(artifacts.fetch("stats.jsonl")) + except ValueError: + return "" + row = stats.get("trial_summary") or next(iter(stats.values()), {}) + return str(row.get("timestamp") or "") + + +def paired_run_finder(dest: str, tag: str, run_ts: int, artifacts: ArtifactStore, tmp: Path, + max_age_s: int = DEFAULT_MAX_AGE_S) -> Callable[[str], BenchmarkRunContext]: + """BenchmarkRunContext.run_of for the run of tag at run_ts under dest, whose artifacts are artifacts. + + The paired run of a test is find_paired_run's, opened under tmp and looked + up once. It must have run on the same cluster as this run when both record + one, checked on each call because this run's stats.jsonl exists only after + the run. Any failure (no such run, a listing or permission error, an + unreadable run or stats.jsonl) raises LookupError saying why. + """ + looked_up: dict[str, BenchmarkRunContext | str] = {} + + def run_of(test: str) -> BenchmarkRunContext: + if test not in looked_up: + try: + looked_up[test] = open_run(find_paired_run(dest, test, tag, run_ts, max_age_s), tmp) + except Exception as e: # pylint: disable=broad-except + # KeyError and IndexError are LookupErrors too, but not ours. + looked_up[test] = str(e) if type(e) is LookupError else f"{type(e).__name__}: {e}" + run = looked_up[test] + if isinstance(run, str): + raise LookupError(run) + try: + mine, theirs = cluster_name(artifacts), cluster_name(run.artifacts) + except (ValueError, OSError) as e: + raise LookupError(f"can't compare the clusters: {e}") from None + if mine and theirs and mine != theirs: + raise LookupError(f"cluster differs ({mine} vs {theirs})") + return run + + return run_of + + +def _list_runs(prefix: str) -> list[str]: + """Run directories (with a logs.txt or stats.jsonl) under prefix, which ends in run_date=/.""" + marks = ("/logs.txt", "/stats.jsonl") + if not prefix.startswith("gs://"): + found = [p for m in marks for p in Path(prefix).glob(f"run_ts=*/run_tag=*{m}")] + return sorted({str(p.parent) for p in found}) + names = _gcs_list(prefix) + return sorted({n.rsplit("/", 1)[0] for n in names if n.endswith(marks)}) + + +def _split(uri: str) -> tuple[str, str]: + bucket, _, path = uri[len("gs://"):].partition("/") + return bucket, path + + +def _storage_client(): + """A google-cloud-storage client, or None when the library is not installed.""" + try: + from google.cloud import storage # pylint: disable=import-outside-toplevel + except ImportError: + return None + return storage.Client() + + +def _gcs_list(prefix: str) -> list[str]: + """gs:// URIs of the objects under prefix. Raises OSError on a listing error.""" + client = _storage_client() + bucket, path = _split(prefix) + if client is not None: + return [f"gs://{bucket}/{b.name}" for b in client.list_blobs(bucket, prefix=path)] + if not shutil.which("gcloud"): + raise OSError("reading gs:// needs google-cloud-storage or gcloud") + res = subprocess.run(["gcloud", "storage", "ls", "-r", prefix], capture_output=True, text=True, check=False) + if res.returncode != 0: + if "matched no objects" in res.stderr or "One or more URLs matched no objects" in res.stderr: + return [] + raise OSError(f"gcloud storage ls {prefix}: {res.stderr.strip()[:300]}") + return [line.strip() for line in res.stdout.splitlines() if line.strip().startswith("gs://")] + + +def _gcs_download(uri: str, dest: Path) -> Optional[Path]: + """Downloads the object uri to dest; None when there is no such object. Never writes to GCS. + Raises OSError on a read error.""" + dest.parent.mkdir(parents=True, exist_ok=True) + client = _storage_client() + if client is not None: + bucket, path = _split(uri) + blob = client.bucket(bucket).get_blob(path) + if blob is None: + return None + blob.download_to_filename(str(dest)) + return dest + if not shutil.which("gcloud"): + raise OSError("reading gs:// needs google-cloud-storage or gcloud") + res = subprocess.run(["gcloud", "storage", "cp", uri, str(dest)], capture_output=True, text=True, check=False) + if res.returncode != 0: + if "matched no objects" in res.stderr or "not found" in res.stderr.lower(): + return None + raise OSError(f"gcloud storage cp {uri}: {res.stderr.strip()[:300]}") + return dest diff --git a/benchmarking/evaluate/sources/__init__.py b/benchmarking/evaluate/sources/__init__.py index c492f932f5..fd7496f0e6 100644 --- a/benchmarking/evaluate/sources/__init__.py +++ b/benchmarking/evaluate/sources/__init__.py @@ -15,6 +15,7 @@ """The sources, by the name a variable's `source` key gives (hooks: base.py).""" from .base import BenchmarkRunContext, Scalar, Source, SourceInvalid # noqa: F401 +from .cel import Cel from .run_results import RunResults -SOURCES: dict[str, type[Source]] = {"run_results": RunResults} +SOURCES: dict[str, type[Source]] = {"cel": Cel, "run_results": RunResults} diff --git a/benchmarking/evaluate/sources/base.py b/benchmarking/evaluate/sources/base.py index a6895ed0aa..d4db869096 100644 --- a/benchmarking/evaluate/sources/base.py +++ b/benchmarking/evaluate/sources/base.py @@ -33,7 +33,7 @@ from dataclasses import dataclass from pathlib import Path -from typing import Annotated, Any, Mapping, Optional, Protocol, TypeVar, Union +from typing import Annotated, Any, Callable, Mapping, Optional, Protocol, TypeVar, Union from pydantic import BaseModel, ConfigDict, Field, ValidationError @@ -67,6 +67,11 @@ def text(err) -> str: for err in e.errors()) +def no_other_runs(test: str) -> "BenchmarkRunContext": + """BenchmarkRunContext.run_of of a run that has no other runs to read.""" + raise LookupError(f"no run of {test}") + + @dataclass(frozen=True) class RunIdentity: """Which run it is. uri is where the run is stored; the other fields are @@ -108,9 +113,16 @@ class BenchmarkRunContext: (identity.uri, artifacts) and its stored artifacts. Each source chooses what it reads: one of the run's artifacts, by name, or another store, keyed by the run's identity. + + run_of(test) returns the run of another test from the same invocation + (e.g. the suite's 1-user run), with its own identity, or raises + LookupError saying why there is none; whoever creates the context says + what "the same invocation" means. Call it from query: finding the run may read + this run's artifacts. """ identity: RunIdentity artifacts: ArtifactStore + run_of: Callable[[str], "BenchmarkRunContext"] = no_other_runs def artifact(self, name: str) -> Optional[Path]: """The local path of the run's stored artifact name, fetched on first diff --git a/benchmarking/evaluate/sources/cel/__init__.py b/benchmarking/evaluate/sources/cel/__init__.py new file mode 100644 index 0000000000..e18ffe6392 --- /dev/null +++ b/benchmarking/evaluate/sources/cel/__init__.py @@ -0,0 +1,233 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The cel source: binds data read from runs to CEL, and evaluates named expressions into values. + + source_cfg: + inputs: # the data: each is a CEL variable + - {name: stats, reader: json_lines, reader_cfg: {file: stats.jsonl}} + - {name: other, from_test: some_other_test, reader: json_lines, reader_cfg: {file: stats.jsonl}} + locals: # optional: ordered list of named in-between results + - {name: rows, cel: "stats.lines.filter(l, l.metric == 'trial_summary')"} + checks: # optional: what must hold for the values to mean anything + - {name: has_rows, cel: "size(rows) > 0"} + values: # the outputs: each a double, bool, string or null + - {name: cores, cel: "size(rows) > 0 ? dyn(rows[size(rows) - 1].measurements.allocatable_cores) : null"} + +An input names a reader (READERS) and its reader_cfg. By default it reads this +run; with `from_test:` it reads that test's run from the same invocation +(BenchmarkRunContext.run_of). Locals are evaluated top to bottom, each +reading inputs and earlier locals by name. The source knows nothing about what +the data means: what is good, which run to compare against and every formula +live in the config. + +At run time, an input that can't be read, a local, check or value that fails +to evaluate, and a check that is false make the variable invalid, and the +reason names which. A value that isn't finite becomes null. +""" + +import json +import math +from dataclasses import dataclass +from typing import Annotated, Any, Mapping, Optional + +from cel_expr_python import cel +from cel_expr_python.ext import ext_math +from pydantic import Field, JsonValue, field_validator, model_validator + +from ... import cel_env +from ..base import BenchmarkRunContext, Model, Name, Scalar, Source, SourceInvalid, parse +from .json_file import Json +from .json_lines import JsonLines +from .reader import Reader, ReadError, check_data + +READERS: dict[str, type[Reader]] = {"json": Json, "json_lines": JsonLines} +VALUE_TYPES = (cel.Type.DOUBLE, cel.Type.BOOL, cel.Type.STRING, cel.Type.NULL) +FUNCTIONS = (cel_env.ROUND, cel_env.PERCENTILE) +EXTENSIONS = (ext_math.ExtMath(),) + + +class Input(Model): + """An input: its name, the test whose run it reads (None: this run), a reader and the reader's config.""" + name: Name + from_test: Optional[Annotated[str, Field(min_length=1)]] = None + reader: str + reader_cfg: dict[str, JsonValue] = {} + + @field_validator("reader") + @classmethod + def _known(cls, reader: str) -> str: + if reader not in READERS: + raise ValueError(f"reader {reader!r} is not one of {sorted(READERS)}") + return reader + + @model_validator(mode="after") + def _reader_cfg(self) -> "Input": + try: + parse(READERS[self.reader].Config, self.reader_cfg) + except ValueError as e: + raise ValueError(f"reader_cfg: {e}") from None + return self + + +class NamedCel(Model): + """A local, check or value: a name and a CEL expression.""" + name: Name + cel: str + + +class Config(Model): + # Lists, not tuples, as in config.Evaluation. + inputs: list[Input] = Field(min_length=1) + locals: list[NamedCel] = [] + checks: list[NamedCel] = [] + values: list[NamedCel] = Field(min_length=1) + + +@dataclass(frozen=True) +class Expression: + kind: str # "local", "check" or "value", for error messages + name: str + expr: cel.Expression + + +@dataclass(frozen=True) +class Program: + """A source_cfg, compiled: the inputs, the locals in list order, the checks and the values.""" + inputs: tuple[Input, ...] + locals: tuple[Expression, ...] + checks: tuple[Expression, ...] + values: tuple[Expression, ...] + + +def _unique(names: list[str], where: str) -> None: + if repeated := sorted({n for n in names if names.count(n) > 1}): + raise ValueError(f"{where}: names repeated: {repeated}") + + +def compile_cfg(cfg: Mapping[str, Any]) -> Program: + """The program of cfg. Raises ValueError for a bad cfg.""" + c = parse(Config, cfg) + input_names = [i.name for i in c.inputs] + local_names = [local.name for local in c.locals] + _unique(input_names, "inputs") + _unique(local_names, "locals") + _unique([check.name for check in c.checks], "checks") + _unique([value.name for value in c.values], "values") + if clash := sorted(set(input_names) & set(local_names)): + raise ValueError(f"locals: {clash} are also inputs") + + # Compile locals top-to-bottom so each local can read inputs and earlier + # locals with their inferred CEL return type (including TIMESTAMP/DURATION). + var_types: dict[str, cel.Type] = {name: cel.Type.Map(cel.Type.STRING, cel.Type.DYN) for name in input_names} + local_exprs: list[Expression] = [] + for local in c.locals: + expr = cel_env.Env(var_types, FUNCTIONS, EXTENSIONS).compile(local.cel, f"local {local.name!r}", None) + var_types[local.name] = expr.return_type() + local_exprs.append(Expression(kind="local", name=local.name, expr=expr)) + + env = cel_env.Env(var_types, FUNCTIONS, EXTENSIONS) + + def compiled(kind: str, section: list[NamedCel], gives: tuple[cel.Type, ...]) -> tuple[Expression, ...]: + return tuple(Expression(kind=kind, name=n.name, expr=env.compile(n.cel, f"{kind} {n.name!r}", gives)) + for n in section) + + return Program( + inputs=tuple(c.inputs), + locals=tuple(local_exprs), + checks=compiled("check", c.checks, (cel.Type.BOOL, cel.Type.DYN)), + values=compiled("value", c.values, (*VALUE_TYPES, cel.Type.DYN)), + ) + + +def run_program(program: Program, data: dict[str, Any]) -> dict[str, Scalar]: + """The values of program over data (the inputs' data by input name). + Raises ValueError, naming the expression, when a local, check or value + fails to evaluate, or a check is false.""" + data = dict(data) + for local in program.locals: + data[local.name] = _eval(local, data) + for check in program.checks: + ok = _eval(check, data) + if not isinstance(ok, bool): + raise ValueError(f"check {check.name!r} gives a {type(ok).__name__}: it must give a bool") + if not ok: + raise ValueError(f"check {check.name!r} is false") + values: dict[str, Scalar] = {} + for value in program.values: + x = _eval(value, data) + if x is not None and type(x) not in (float, bool, str): # pylint: disable=unidiomatic-typecheck # int is not a double + raise ValueError( + f"value {value.name!r} gives a {type(x).__name__}: it must give a double, bool, string or null") + values[value.name] = None if isinstance(x, float) and not math.isfinite(x) else x + return values + + +def _eval(e: Expression, data: Mapping[str, Any]) -> Any: + try: + result = e.expr.eval(data=data) + except RuntimeError as err: # a value CEL can't take in, such as a string with a lone surrogate + raise ValueError(f"{e.kind} {e.name!r} failed to evaluate: {err}") from None + if result.type() == cel.Type.ERROR: + raise ValueError(f"{e.kind} {e.name!r} failed to evaluate: {result.value()}") + return result.plain_value() # lists and maps as Python ones, which a later expression can read + + +class Cel(Source): + @classmethod + def validate(cls, cfg: Mapping[str, Any]) -> None: + compile_cfg(cfg) + + def init(self, run: BenchmarkRunContext) -> None: + super().init(run) + self._runs: dict[str, BenchmarkRunContext | ReadError] = {} + self._reads: dict[tuple[Optional[str], str, str], Mapping[str, Any] | ReadError] = {} + + def query(self, cfg: Mapping[str, Any]) -> Mapping[str, Scalar]: + program = compile_cfg(cfg) + data = {i.name: self._read(i) for i in program.inputs} + try: + return run_program(program, data) + except ValueError as e: + raise SourceInvalid(str(e)) from None + + def _read(self, i: Input) -> Mapping[str, Any]: + """The data of input i, read once per run and reader config; SourceInvalid naming it when it can't be read.""" + key = (i.from_test, i.reader, json.dumps(i.reader_cfg, sort_keys=True)) + if key not in self._reads: + try: + reader = READERS[i.reader] + data = reader().read(self._run(i.from_test), parse(reader.Config, i.reader_cfg)) + check_data(reader, data) + self._reads[key] = data + except ReadError as e: + self._reads[key] = e + found = self._reads[key] + if isinstance(found, ReadError): + raise SourceInvalid(f"input {i.name!r}: {found}") + return found + + def _run(self, test: Optional[str]) -> BenchmarkRunContext: + """The run of test (None: this run); ReadError when there is none.""" + if test is None: + return self.run + if test not in self._runs: + try: + self._runs[test] = self.run.run_of(test) + except LookupError as e: + self._runs[test] = ReadError(str(e)) + found = self._runs[test] + if isinstance(found, ReadError): + raise found + return found diff --git a/benchmarking/evaluate/sources/cel/json_file.py b/benchmarking/evaluate/sources/cel/json_file.py new file mode 100644 index 0000000000..8fff6096d6 --- /dev/null +++ b/benchmarking/evaluate/sources/cel/json_file.py @@ -0,0 +1,53 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""json: one JSON object file from the run. + + - {name: summary, reader: json, reader_cfg: {file: server_summary.json}} + + summary.data the parsed JSON object, with leaf coercion as in run_results + summary.uri where the run is stored +""" + +import json +from typing import Annotated, Any, Mapping + +import pydantic + +from ..base import BenchmarkRunContext, Model +from .json_lines import coerce_json +from .reader import Field, Reader, ReadError, to_data + + +class Config(Model): + file: Annotated[str, pydantic.Field(min_length=1)] # the name of one of the run's files + + +class Json(Reader): + fields = (Field("data", "map"), Field("uri", "string")) + Config = Config + + def read(self, run: BenchmarkRunContext, cfg: Config) -> Mapping[str, Any]: + name = cfg.file + path = run.artifact(name) + if path is None: + raise ReadError(f"{name} absent") + with open(path, encoding="utf-8", errors="replace") as f: + try: + parsed = json.load(f) + except ValueError as e: + raise ReadError(f"{name}: not JSON: {e}") from None + if not isinstance(parsed, dict): + raise ReadError(f"{name}: not a JSON object") + return to_data({"data": coerce_json(parsed), "uri": run.identity.uri}) diff --git a/benchmarking/evaluate/sources/cel/json_lines.py b/benchmarking/evaluate/sources/cel/json_lines.py new file mode 100644 index 0000000000..b8471bf892 --- /dev/null +++ b/benchmarking/evaluate/sources/cel/json_lines.py @@ -0,0 +1,157 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""json_lines: filtered lines of one of the run's JSON-lines or prefixed-JSON log files. + + - {name: stats, reader: json_lines, reader_cfg: {file: stats.jsonl}} + - name: records + reader: json_lines + reader_cfg: + file: logs.txt + prefix: "[boomer] " + until: "has(line.msg) && line.msg == 'shutdown'" + select: "has(line.msg) && line.msg == 'record'" + unique_by: actor + + .lines selected JSON objects, in file order + .unparsed_lines [{line_no, text, error}] for prefixed (or non-blank) lines that aren't JSON objects + .uri where the run is stored + +reader_cfg, besides `file`: +- `prefix`: skip lines without this prefix, and strip it before parsing JSON. + Without `prefix`, blank lines are skipped. +- `until`: CEL boolean expression over `line`; stop reading when a parsed + object matches (that line is not emitted). +- `select`: CEL boolean expression over `line`; keep only matching objects. +- `unique_by`: field name; keep only the first selected object per distinct + value of that field. + +Values are coerced as in run_results: numbers and numeric strings become +doubles, "true"/"false" become bools, and a null or non-finite number is +left out, so has() tests for it. +""" + +import json +from typing import Annotated, Any, Mapping, Optional + +import pydantic +from cel_expr_python import cel +from cel_expr_python.ext import ext_math + +from ... import cel_env +from ..base import BenchmarkRunContext, Model +from ..run_results import coerce +from .reader import Field, Reader, ReadError, to_data + + +def coerce_json(v: Any) -> Any: + """A JSON value with coerce applied to each leaf.""" + if isinstance(v, dict): + return {k: coerce_json(x) for k, x in v.items()} + if isinstance(v, list): + return [coerce_json(x) for x in v] + return coerce(v) + + +def _line_env() -> cel_env.Env: + return cel_env.Env( + {"line": cel.Type.Map(cel.Type.STRING, cel.Type.DYN)}, + extensions=(ext_math.ExtMath(),), + ) + + +def _compile_line_expr(raw: str, where: str) -> cel.Expression: + return _line_env().compile(raw, where, (cel.Type.BOOL, cel.Type.DYN)) + + +def _eval_line_bool(expr: cel.Expression, line: Mapping[str, Any], line_no: int, where: str) -> bool: + try: + result = expr.eval(data={"line": line}) + except RuntimeError as e: + raise ReadError(f"line {line_no}: {where} failed: {e}") from None + if result.type() != cel.Type.BOOL: + detail = result.value() if result.type() == cel.Type.ERROR else f"gave {result.type().name()}" + raise ReadError(f"line {line_no}: {where} failed: {detail}") + return result.value() + + +NonEmpty = Annotated[str, pydantic.Field(min_length=1)] + + +class Config(Model): + file: NonEmpty # the name of one of the run's files + prefix: str = "" + select: Optional[str] = None + until: Optional[str] = None + unique_by: Optional[NonEmpty] = None # a field name + + @pydantic.field_validator("select", "until") + @classmethod + def _compiles(cls, expression: Optional[str], info: pydantic.ValidationInfo) -> Optional[str]: + if expression is not None: + _compile_line_expr(expression, info.field_name) + return expression + + +class JsonLines(Reader): + fields = (Field("lines", "list"), Field("unparsed_lines", "list"), Field("uri", "string")) + Config = Config + + def read(self, run: BenchmarkRunContext, cfg: Config) -> Mapping[str, Any]: + o = cfg + name, prefix, unique_by = o.file, o.prefix, o.unique_by + path = run.artifact(name) + if path is None: + raise ReadError(f"{name} absent") + until_expr = None if o.until is None else _compile_line_expr(o.until, "until") + select_expr = None if o.select is None else _compile_line_expr(o.select, "select") + + lines, unparsed = [], [] + seen: set[tuple[type, Any]] = set() + with open(path, encoding="utf-8", errors="replace") as f: + for n, text in enumerate(f, 1): + raw = text.rstrip("\n") + if prefix: + if not raw.startswith(prefix): + continue + payload = raw[len(prefix):] + else: + if not raw.strip(): + continue + payload = raw + try: + parsed = json.loads(payload) + error = "" if isinstance(parsed, dict) else "not a JSON object" + except ValueError as e: + error = f"not JSON: {e}" + if error: + unparsed.append({"line_no": n, "text": raw, "error": error}) + continue + coerced = to_data(coerce_json(parsed)) + if until_expr is not None and _eval_line_bool(until_expr, coerced, n, "until"): + break + if select_expr is not None and not _eval_line_bool(select_expr, coerced, n, "select"): + continue + if unique_by is not None: + if unique_by not in coerced: + raise ReadError(f"line {n}: unique_by {unique_by!r}: field missing") + val = coerced[unique_by] + if isinstance(val, (dict, list)): + raise ReadError(f"line {n}: unique_by {unique_by!r}: unhashable {type(val).__name__}") + dedup = (type(val), val) + if dedup in seen: + continue + seen.add(dedup) + lines.append(coerced) + return to_data({"lines": lines, "unparsed_lines": unparsed, "uri": run.identity.uri}) diff --git a/benchmarking/evaluate/sources/cel/reader.py b/benchmarking/evaluate/sources/cel/reader.py new file mode 100644 index 0000000000..2fd0f2cee1 --- /dev/null +++ b/benchmarking/evaluate/sources/cel/reader.py @@ -0,0 +1,85 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""What the cel source needs from a reader: a class that turns one run's files into CEL data. + +A reader exposes data and passes no judgment on it. Its data is a mapping +with the top-level fields it declares; lists hold maps or doubles, and a +field a record lacks is absent rather than null, so has() tests for it. +""" + +import dataclasses +import math +from dataclasses import dataclass +from typing import Any, ClassVar, Literal, Mapping + +from ..base import BenchmarkRunContext, Model + +FieldType = Literal["double", "string", "bool", "list", "map"] +PY_TYPES: dict[str, type | tuple[type, ...]] = {"double": float, "string": str, "bool": bool, "list": list, + "map": dict} + + +class ReadError(Exception): + """The reader can't read the run: why (e.g. the file is absent).""" + + +@dataclass(frozen=True) +class Field: + """A top-level field of a reader's data, with its type, checked by check_data.""" + name: str + type: FieldType + optional: bool = False + + +class Reader: + """A reader; the cel source makes one per input that names it.""" + + fields: ClassVar[tuple[Field, ...]] + Config: ClassVar[type[Model]] # the input's reader_cfg; validating it raises ValueError for a bad one + + def read(self, run: BenchmarkRunContext, cfg: Any) -> Mapping[str, Any]: + """The run's data, read as cfg (a Config) says. Raises ReadError when the run can't be read.""" + raise NotImplementedError + + +def check_data(reader: type[Reader], data: Mapping[str, Any]) -> None: + """Raises TypeError unless data has the reader's fields, of their types, and no others.""" + declared = {f.name: f for f in reader.fields} + if unknown := set(data) - set(declared): + raise TypeError(f"{reader.__name__} gave undeclared fields {sorted(unknown)}") + for f in reader.fields: + if f.name not in data: + if not f.optional: + raise TypeError(f"{reader.__name__} gave no {f.name}") + elif isinstance(data[f.name], bool) != (f.type == "bool") or not isinstance(data[f.name], PY_TYPES[f.type]): + raise TypeError(f"{reader.__name__}: {f.name} is a {type(data[f.name]).__name__}, not a {f.type}") + + +def to_data(obj: Any) -> Any: + """A dataclass (or list or dict of them) as CEL data: dicts and lists, + numbers as doubles, a None or non-finite field left out.""" + if dataclasses.is_dataclass(obj) and not isinstance(obj, type): + obj = {f.name: getattr(obj, f.name) for f in dataclasses.fields(obj)} + if isinstance(obj, dict): + return {k: to_data(v) for k, v in obj.items() if _present(v)} + if isinstance(obj, (list, tuple)): + return [to_data(v) for v in obj if _present(v)] + if isinstance(obj, int) and not isinstance(obj, bool): + return float(obj) + return obj + + +def _present(v: Any) -> bool: + return v is not None and not (isinstance(v, float) and not math.isfinite(v)) diff --git a/benchmarking/evaluate/unit_tests/test_cel.py b/benchmarking/evaluate/unit_tests/test_cel.py new file mode 100644 index 0000000000..be87a40e98 --- /dev/null +++ b/benchmarking/evaluate/unit_tests/test_cel.py @@ -0,0 +1,451 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the cel source, its CEL helpers, and the json / json_lines readers.""" + +import dataclasses +import json +import math +import sys +import tempfile +import unittest +from pathlib import Path +from typing import Optional +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from cel_expr_python import cel as cel_lib # noqa: E402 +from cel_expr_python.ext import ext_math # noqa: E402 + +from evaluate import cel_env # noqa: E402 +from evaluate.sources import cel # noqa: E402 +from evaluate.sources.base import BenchmarkRunContext, LocalArtifacts, RunIdentity, SourceInvalid # noqa: E402 +from evaluate.sources.cel import json_file, json_lines, reader # noqa: E402 + +def stats_input(name: str = "stats", **reader_cfg) -> dict: + """An input reading stats.jsonl with json_lines, with more reader_cfg.""" + return {"name": name, "reader": "json_lines", "reader_cfg": {"file": "stats.jsonl", **reader_cfg}} + + +STATS = [stats_input()] + + +def stats_row(metric: str, **measurements) -> dict: + return {"timestamp": "2026-10-24T11:16:49Z", "tag": "abc", "test_name": "t", "metric": metric, + "measurements": measurements} + + +def local_run(d: Path, uri: Optional[str] = None) -> BenchmarkRunContext: + return BenchmarkRunContext(RunIdentity(uri or str(d)), LocalArtifacts(d)) + + +NO_RUN = local_run(Path(tempfile.gettempdir()) / "evaluate-unit-tests-no-run") # holds no files + + +def read_lines(run: BenchmarkRunContext, reader_cfg: dict) -> dict: + return dict(json_lines.JsonLines().read(run, json_lines.Config(**reader_cfg))) + + +def read_json(run: BenchmarkRunContext, reader_cfg: dict) -> dict: + return dict(json_file.Json().read(run, json_file.Config(**reader_cfg))) + + +def write_stats(d: Path, lines: list, *, summary: Optional[object] = None) -> BenchmarkRunContext: + """A run whose stats.jsonl holds lines (dicts are dumped as JSON), plus optional summary.json.""" + d.mkdir(parents=True, exist_ok=True) + path = d / "stats.jsonl" + path.write_text("".join((json.dumps(x) if isinstance(x, dict) else x) + "\n" for x in lines), encoding="utf-8") + if summary is not None: + spath = d / "summary.json" + spath.write_text(json.dumps(summary) if isinstance(summary, (dict, list)) else str(summary), encoding="utf-8") + return local_run(d) + + +def query(run: BenchmarkRunContext, cfg: dict) -> tuple[dict, Optional[str]]: + """One query of run through the hooks: the values, and why the variable is invalid (None when valid).""" + cel.Cel.validate(cfg) + source = cel.Cel() + source.init(run) + try: + return dict(source.query(cfg)), None + except SourceInvalid as e: + return e.values, e.reason + + +def query_lines(lines: list, cfg: dict) -> tuple[dict, Optional[str]]: + with tempfile.TemporaryDirectory() as tmp: + return query(write_stats(Path(tmp), lines), {"inputs": STATS, **cfg}) + + +def evaluate(expression: str, **data): + """expression over data, with the cel source's functions and math extension.""" + env = cel_env.Env( + {k: cel_lib.Type.DYN for k in data}, + (cel_env.ROUND, cel_env.PERCENTILE), + extensions=(ext_math.ExtMath(),), + ) + result = env.compile(expression, "test", None).eval(data=data) + if result.type() == cel_lib.Type.ERROR: + raise ValueError(result.value()) + return result.plain_value() + + +class FunctionsTest(unittest.TestCase): + def test_percentile(self): + self.assertEqual(evaluate("percentile(xs, 50.0)", xs=[4.0, 1.0, 3.0, 2.0]), 2.5) + self.assertEqual(evaluate("percentile(xs, 90.0)", xs=[1.0, 2.0, 3.0, 4.0, 5.0]), 4.6) + self.assertEqual(evaluate("percentile(xs, 0.0)", xs=[3.0, 1.0]), 1.0) + self.assertEqual(evaluate("percentile(xs, 100.0)", xs=[3.0, 1.0]), 3.0) + self.assertEqual(evaluate("percentile(xs, 99.0)", xs=[7.0]), 7.0) + with self.assertRaisesRegex(ValueError, "empty list"): + evaluate("percentile(xs, 50.0)", xs=[]) + with self.assertRaisesRegex(ValueError, "non-finite"): + evaluate("percentile(xs, 50.0)", xs=[1.0, float("nan")]) + with self.assertRaisesRegex(ValueError, "between 0 and 100"): + evaluate("percentile(xs, 101.0)", xs=[1.0]) + + def test_ext_math(self): + self.assertEqual(evaluate("[math.least(xs), math.greatest(xs)]", xs=[2.0, -1.0, 5.0]), [-1.0, 5.0]) + self.assertEqual(evaluate("math.least(2.0, -1.0)"), -1.0) + with self.assertRaises(ValueError): + evaluate("math.least(xs)", xs=[]) + + def test_a_list_of_another_type_is_an_error(self): + with self.assertRaises(ValueError): + evaluate("percentile(xs, 50.0)", xs=["a"]) + + +class ConfigTest(unittest.TestCase): + VALID = {"inputs": STATS, "locals": [{"name": "rows", "cel": "stats.lines"}], + "checks": [{"name": "some", "cel": "size(rows) > 0"}], + "values": [{"name": "n", "cel": "double(size(rows))"}]} + + def test_valid(self): + cel.Cel.validate(self.VALID) + cel.Cel.validate({"inputs": STATS, "values": [{"name": "n", "cel": "double(size(stats.lines))"}]}) + cel.Cel.validate({"inputs": STATS, "values": [{"name": "n", "cel": "size(stats.lines) > 0 ? dyn(1.0) : null"}]}) + + def test_invalid(self): + cases = { + "rules: Extra inputs are not permitted": {**self.VALID, "rules": []}, + "inputs: Field required": {"values": [{"name": "n", "cel": "1.0"}]}, + "inputs: List should have at least 1 item": {**self.VALID, "inputs": []}, + "inputs.0.name: String should match pattern": {**self.VALID, "inputs": [stats_input("Stats")]}, + "inputs: names repeated: \\['stats'\\]": {**self.VALID, "inputs": [stats_input(), stats_input()]}, + "inputs.0.file: Extra inputs are not permitted": {**self.VALID, "inputs": [ + {"name": "stats", "reader": "json_lines", "file": "stats.jsonl"}]}, + "locals: Input should be a valid list": {**self.VALID, "locals": {"rows": "stats.lines"}}, + "locals.0.name: String should match pattern": {**self.VALID, "locals": [ + {"name": "Rows", "cel": "stats.lines"}]}, + "locals: names repeated: \\['rows'\\]": {**self.VALID, "locals": [ + {"name": "rows", "cel": "stats.lines"}, {"name": "rows", "cel": "stats.lines"}]}, + "locals.0.extra: Extra inputs are not permitted": {**self.VALID, "locals": [ + {"name": "rows", "cel": "stats.lines", "extra": 1}]}, + "reader 'csv' is not one of": {**self.VALID, "inputs": [{"name": "stats", "reader": "csv"}]}, + "inputs.0: reader_cfg: file: Field required": {**self.VALID, "inputs": [ + {"name": "stats", "reader": "json_lines"}]}, + "inputs.0: reader_cfg: unknown: Extra inputs are not permitted": { + **self.VALID, "inputs": [stats_input(unknown="true")]}, + "inputs.0: reader_cfg: prefix: Input should be a valid string": { + **self.VALID, "inputs": [stats_input(prefix=1)]}, + "inputs.0: reader_cfg: unique_by: String should have at least 1 character": { + **self.VALID, "inputs": [stats_input(unique_by="")]}, + "inputs.0: reader_cfg: select: .*Syntax error": { + **self.VALID, "inputs": [stats_input(select="line.a +")]}, + "inputs.0: reader_cfg: until: .*must give a bool": { + **self.VALID, "inputs": [stats_input(until="size(line)")]}, + "inputs.0.from_test: String should have at least 1 character": { + **self.VALID, "inputs": [{**stats_input(), "from_test": ""}]}, + "locals: \\['stats'\\] are also inputs": {**self.VALID, "locals": [ + {"name": "stats", "cel": "1.0"}, {"name": "rows", "cel": "stats"}]}, + "local 'rows': .*Syntax error": {**self.VALID, "locals": [ + {"name": "in", "cel": "1.0"}, {"name": "rows", "cel": "in"}]}, + "variable 'int' declared multiple times": {**self.VALID, "locals": [ + {"name": "int", "cel": "1.0"}, {"name": "rows", "cel": "stats.lines"}]}, + "check 'some'": {**self.VALID, "checks": [{"name": "some", "cel": "size(rows) +"}]}, + "checks: names repeated: \\['some'\\]": {**self.VALID, "checks": [ + {"name": "some", "cel": "true"}, {"name": "some", "cel": "true"}]}, + "values: names repeated: \\['n'\\]": {**self.VALID, "values": [ + {"name": "n", "cel": "1.0"}, {"name": "n", "cel": "2.0"}]}, + "values: List should have at least 1 item": {**self.VALID, "values": []}, + "check 'some': must give a bool": {**self.VALID, "checks": [{"name": "some", "cel": "size(rows)"}]}, + "value 'n': must give a double or a bool or a string or a null": { + **self.VALID, "values": [{"name": "n", "cel": "size(rows)"}]}, + "undeclared reference to 'nope'": {**self.VALID, "values": [{"name": "n", "cel": "nope"}]}, + "undeclared reference to 'later'": {**self.VALID, "locals": [ + {"name": "rows", "cel": "later"}, {"name": "later", "cel": "stats.lines"}]}, + "undeclared reference to 'rows'": {**self.VALID, "locals": [ + {"name": "rows", "cel": "rows"}]}, + "value 'n': .*no matching overload": {**self.VALID, "values": [ + {"name": "n", "cel": "double(size(rows)) / 2"}]}, + } + for want, cfg in cases.items(): + with self.subTest(want=want), self.assertRaisesRegex(ValueError, want): + cel.Cel.validate(cfg) + + def test_inputs_and_locals_need_not_be_read(self): + """Config load accepts inputs and locals that nothing reads, so entries can share them through YAML anchors.""" + cel.Cel.validate({**self.VALID, "inputs": [*STATS, stats_input("other")]}) + cel.Cel.validate({**self.VALID, "checks": [], "values": [{"name": "n", "cel": "1.0"}]}) + + def test_a_failed_local_makes_the_variable_invalid_even_if_unread(self): + """Like an input that can't be read: every local must evaluate.""" + locals_ = [{"name": "rows", "cel": "stats.lines"}, + {"name": "sixth", "cel": "rows[5]"}] + v, why = query_lines([stats_row("trial_summary")], + {"locals": locals_, "values": [{"name": "n", "cel": "double(size(rows))"}]}) + self.assertEqual(v, {}) + self.assertRegex(why, "^local 'sixth' failed to evaluate: .*Index error") + + +class QueryTest(unittest.TestCase): + LINES = [stats_row("trial_summary", allocatable_cores="192"), stats_row("grpc_ResumeActor", failure_count=2), + stats_row("trial_summary", allocatable_cores=88)] + + def test_locals_checks_and_values(self): + v, why = query_lines(self.LINES, { + "locals": [ + {"name": "trials", "cel": "stats.lines.filter(l, l.metric == 'trial_summary')"}, + {"name": "cores", "cel": "trials.map(l, l.measurements.allocatable_cores)"}, + {"name": "last", "cel": "cores[size(cores) - 1]"}, + ], + "checks": [{"name": "has_trial", "cel": "size(trials) > 0"}], + "values": [ + {"name": "cores", "cel": "last"}, + {"name": "most", "cel": "math.greatest(cores)"}, + {"name": "name", "cel": "trials[0].metric"}, + {"name": "many", "cel": "size(cores) > 1"}, + {"name": "per_zero", "cel": "last / 0.0"}, + {"name": "nullable", "cel": "size(cores) > 5 ? dyn(last) : null"}, + ], + }) + self.assertIsNone(why) + self.assertEqual(v, {"cores": 88.0, "most": 192.0, "name": "trial_summary", "many": True, + "per_zero": None, "nullable": None}) + + def test_timestamps_and_durations_in_locals(self): + v, why = query_lines([ + {"time": "2026-10-11T07:00:00.250Z", "val": 10.0}, + {"time": "2026-10-11T07:01:00.750Z", "val": 20.0}, + {"time": "2026-10-11T07:03:00.000Z", "val": 30.0}, + ], { + "locals": [ + {"name": "lo", "cel": "timestamp(int(1791702000.0))"}, + {"name": "hi", "cel": "timestamp('2026-10-11T07:02:00Z')"}, + {"name": "span", "cel": "hi - lo"}, + {"name": "kept", "cel": "stats.lines.filter(r, timestamp(r.time) >= lo && timestamp(r.time) <= hi)"}, + ], + "values": [ + {"name": "count", "cel": "double(size(kept))"}, + {"name": "span_s", "cel": "double(span.getSeconds())"}, + ], + }) + self.assertIsNone(why) + self.assertEqual(v, {"count": 2.0, "span_s": 120.0}) + + def test_a_false_check_makes_the_variable_invalid(self): + v, why = query_lines(self.LINES, {"checks": [{"name": "no_failures", "cel": "stats.lines.all(l, !has(l.measurements.failure_count)" + " || l.measurements.failure_count == 0.0)"}], + "values": [{"name": "n", "cel": "double(size(stats.lines))"}]}) + self.assertEqual((v, why), ({}, "check 'no_failures' is false")) + + def test_errors_make_the_variable_invalid_and_name_the_expression(self): + cases = { + "check 'c' failed to evaluate: .*Key not found": ( + {"checks": [{"name": "c", "cel": "stats.lines.all(l, l.measurements.allocatable_cores > 0.0)"}]}, {}), + "local 'l' failed to evaluate: .*Key not found": ( + {"locals": [{"name": "l", "cel": "stats.lines.map(l, l.measurements.allocatable_cores)"}]}, + [{"name": "n", "cel": "math.greatest(l)"}]), + "value 'n' failed to evaluate: .*Key not found": ( + {}, [{"name": "n", "cel": "math.greatest(stats.lines.map(l, l.measurements.allocatable_cores))"}]), + "value 'n' failed to evaluate: .*empty list": ( + {}, [{"name": "n", "cel": "percentile(stats.lines.filter(l, false).map(l, 1.0), 50.0)"}]), + } + for want, (cfg, values) in cases.items(): + with self.subTest(want=want): + v, why = query_lines(self.LINES, + {**cfg, "values": values or [{"name": "n", "cel": "double(size(stats.lines))"}]}) + self.assertEqual(v, {}) + self.assertRegex(why, want) + + def test_an_input_that_cant_be_read_makes_the_variable_invalid(self): + with tempfile.TemporaryDirectory() as tmp: + run = local_run(Path(tmp)) + cfg = {"inputs": STATS, "values": [{"name": "n", "cel": "double(size(stats.lines))"}]} + self.assertEqual(query(run, cfg), ({}, "input 'stats': stats.jsonl absent")) + other = {"inputs": [{**stats_input("other"), "from_test": "t1"}], + "values": [{"name": "n", "cel": "double(size(other.lines))"}]} + self.assertEqual(query(write_stats(Path(tmp), self.LINES), other), + ({}, "input 'other': no run of t1")) + + def test_an_input_with_from_test_reads_that_tests_run(self): + with tempfile.TemporaryDirectory() as tmp: + this = write_stats(Path(tmp) / "this", self.LINES) + paired = write_stats(Path(tmp) / "t1", [stats_row("trial_summary", allocatable_cores=4)]) + asked = [] + + def run_of(test: str) -> BenchmarkRunContext: + asked.append(test) + return paired + + cfg = {"inputs": [*STATS, {**stats_input("other"), "from_test": "t1"}], + "values": [{"name": "ratio", "cel": "stats.lines[0].measurements.allocatable_cores / " + "other.lines[0].measurements.allocatable_cores"}, + {"name": "other_uri", "cel": "other.uri"}]} + self.assertEqual(query(dataclasses.replace(this, run_of=run_of), cfg), + ({"ratio": 48.0, "other_uri": str(Path(tmp) / "t1")}, None)) + self.assertEqual(asked, ["t1"]) + + def test_each_input_is_read_once_per_run(self): + cfg = {"inputs": STATS, "values": [{"name": "n", "cel": "double(size(stats.lines))"}]} + other = {"inputs": [stats_input("s")], "values": [{"name": "m", "cel": "double(size(s.lines))"}]} + with tempfile.TemporaryDirectory() as tmp: + source = cel.Cel() + source.init(write_stats(Path(tmp), self.LINES)) + with mock.patch.object(json_lines.JsonLines, "read", autospec=True, + side_effect=json_lines.JsonLines.read) as read: + self.assertEqual(source.query(cfg), {"n": 3.0}) + self.assertEqual(source.query(other), {"m": 3.0}) + self.assertEqual(read.call_count, 1) + + def test_a_reader_bug_is_not_reported_as_an_unreadable_input(self): + with tempfile.TemporaryDirectory() as tmp: + source = cel.Cel() + source.init(write_stats(Path(tmp), self.LINES)) + with mock.patch.object(json_lines.JsonLines, "read", side_effect=KeyError("bug")), \ + self.assertRaises(KeyError): + source.query({"inputs": STATS, "values": [{"name": "n", "cel": "1.0"}]}) + + def test_init_reads_nothing(self): + source = cel.Cel() + with mock.patch.object(json_lines.JsonLines, "read") as read: + source.init(NO_RUN) + read.assert_not_called() + + +class JsonLinesTest(unittest.TestCase): + def test_lines_are_coerced_as_in_run_results(self): + with tempfile.TemporaryDirectory() as tmp: + data = read_lines(write_stats(Path(tmp), [ + {"metric": "m", "measurements": {"n": "12.5", "i": 3, "flag": "true", "s": "2026_11", "none": None, + "nan": "NaN"}, "list": [1, "2"]}, + "", "not json", "[1, 2]", + ]), {"file": "stats.jsonl"}) + self.assertEqual(data["lines"], [{"metric": "m", "measurements": {"n": 12.5, "i": 3.0, "flag": True, + "s": "2026_11"}, "list": [1.0, 2.0]}]) + self.assertEqual([(u["line_no"], u["text"]) for u in data["unparsed_lines"]], [(3.0, "not json"), (4.0, "[1, 2]")]) + self.assertRegex(data["unparsed_lines"][0]["error"], "^not JSON: ") + self.assertEqual(data["unparsed_lines"][1]["error"], "not a JSON object") + self.assertTrue(data["uri"]) + + def test_prefix_until_select_and_unique_by(self): + with tempfile.TemporaryDirectory() as tmp: + d = Path(tmp) + log = d / "logs.txt" + log.write_text("\n".join([ + "plain runner line not json", + '[boomer] {"msg": "fail", "actor": "a1", "step": "resume"}', + '[boomer] {"msg": "fail", "actor": "a1", "step": "execute"}', + '[boomer] {"msg": "other", "actor": "a9"}', + '[boomer] {"msg": "fail", "actor": "a2", "step": "suspend"}', + '[boomer] {"msg": "shutdown"}', + '[boomer] {"msg": "fail", "actor": "a3", "step": "resume"}', + ]) + "\n", encoding="utf-8") + run = local_run(log.parent, uri="run1") + data = read_lines(run, { + "file": "logs.txt", + "prefix": "[boomer] ", + "until": "has(line.msg) && line.msg == 'shutdown'", + "select": "has(line.msg) && line.msg == 'fail'", + "unique_by": "actor", + }) + self.assertEqual(data["lines"], [ + {"msg": "fail", "actor": "a1", "step": "resume"}, + {"msg": "fail", "actor": "a2", "step": "suspend"}, + ]) + self.assertEqual(data["unparsed_lines"], []) + self.assertEqual(data["uri"], "run1") + + def test_unique_by_distinguishes_bool_and_int(self): + with tempfile.TemporaryDirectory() as tmp: + data = read_lines(write_stats(Path(tmp), [ + {"k": True, "n": 1}, {"k": 1, "n": 2}, {"k": True, "n": 3}, + ]), {"file": "stats.jsonl", "unique_by": "k"}) + self.assertEqual([x["n"] for x in data["lines"]], [1.0, 2.0]) + + def test_errors(self): + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaisesRegex(reader.ReadError, "stats.jsonl absent"): + read_lines(NO_RUN, {"file": "stats.jsonl"}) + run = write_stats(Path(tmp), [{"a": {"nested": 1}}]) + with self.assertRaisesRegex(reader.ReadError, "line 1: select failed: .*Key not found"): + read_lines(run, {"file": "stats.jsonl", "select": "line.missing == 1.0"}) + with self.assertRaisesRegex(reader.ReadError, "line 1: unique_by 'missing': field missing"): + read_lines(run, {"file": "stats.jsonl", "unique_by": "missing"}) + with self.assertRaisesRegex(reader.ReadError, "line 1: unique_by 'a': unhashable dict"): + read_lines(run, {"file": "stats.jsonl", "unique_by": "a"}) + null_run = write_stats(Path(tmp) / "null", [{"a": None, "b": 1}, {"a": "x", "b": 2}]) + selected = read_lines(null_run, {"file": "stats.jsonl", "select": "has(line.a)"}) + self.assertEqual(selected["lines"], [{"a": "x", "b": 2.0}]) + with self.assertRaisesRegex(reader.ReadError, "line 1: unique_by 'a': field missing"): + read_lines(null_run, {"file": "stats.jsonl", "unique_by": "a"}) + + +class JsonFileTest(unittest.TestCase): + def test_reads_and_coerces_json_object(self): + with tempfile.TemporaryDirectory() as tmp: + run = write_stats(Path(tmp), [], summary={"metadata": {"steady_start_ts": "100", "ok": "true", "x": None}}) + data = read_json(run, {"file": "summary.json"}) + self.assertEqual(data["data"], {"metadata": {"steady_start_ts": 100.0, "ok": True}}) + self.assertEqual(data["uri"], str(Path(tmp))) + + def test_errors(self): + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaisesRegex(reader.ReadError, "summary.json absent"): + read_json(NO_RUN, {"file": "summary.json"}) + bad = write_stats(Path(tmp) / "bad", [], summary="{oops") + with self.assertRaisesRegex(reader.ReadError, "summary.json: not JSON: "): + read_json(bad, {"file": "summary.json"}) + arr = write_stats(Path(tmp) / "arr", [], summary=[1, 2]) + with self.assertRaisesRegex(reader.ReadError, "summary.json: not a JSON object"): + read_json(arr, {"file": "summary.json"}) + + +class ReaderTest(unittest.TestCase): + def test_check_data(self): + class R(reader.Reader): + fields = (reader.Field("xs", "list"), reader.Field("n", "double"), reader.Field("ok", "bool", optional=True)) + + reader.check_data(R, {"xs": [], "n": 1.0}) + reader.check_data(R, {"xs": [], "n": 1.0, "ok": False}) + for bad, want in (({"xs": []}, "gave no n"), ({"xs": [], "n": 1.0, "extra": 1.0}, "undeclared fields"), + ({"xs": [], "n": True}, "n is a bool, not a double"), ({"xs": [], "n": 1}, "not a double"), + ({"xs": [], "n": 1.0, "ok": 1.0}, "ok is a float, not a bool")): + with self.subTest(bad=bad), self.assertRaisesRegex(TypeError, want): + reader.check_data(R, bad) + + def test_to_data(self): + @dataclasses.dataclass + class D: + a: int + b: Optional[float] = None + c: float = math.inf + + self.assertEqual(reader.to_data({"d": [D(1)], "n": None, "s": "x", "t": True}), + {"d": [{"a": 1.0}], "s": "x", "t": True}) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarking/evaluate/unit_tests/test_cli.py b/benchmarking/evaluate/unit_tests/test_cli.py new file mode 100644 index 0000000000..0922bb1519 --- /dev/null +++ b/benchmarking/evaluate/unit_tests/test_cli.py @@ -0,0 +1,213 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the CLI, `python -m evaluate`, on stored runs. + +Runs are stand-in directories built by the tests; GCS is mocked. +""" + +import contextlib +import io +import json +import shutil +import sys +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from evaluate import __main__ as cli # noqa: E402 +from evaluate import rundir # noqa: E402 + +T0 = 1791700000 # 2026-10-11T07:06:40Z +TAG = "abc123" + + +def boomer(sec: float, msg: str, **kw) -> str: + t = datetime.fromtimestamp(T0 + sec, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f000Z") + return "[boomer] " + json.dumps({"time": t, "level": "INFO", "msg": msg, **kw}) + + +def record(sec: float, actor: str, wall_ms: float = 8000.0) -> str: + end = datetime.fromtimestamp(T0 + sec, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f000Z") + return boomer(sec, "sweperf task record", v=1, user=1, actor=actor, end=end, cel_ms=6000.0, wall_ms=wall_ms, + resume_max_ms=200.0, ttfe_max_ms=210.0, retries=0, failed=False, fail_step="", fail_cycle=0) + + +def write_run(d: Path, records: list[str], cluster: str = "c1") -> Path: + d.mkdir(parents=True, exist_ok=True) + (d / "logs.txt").write_text("\n".join([*records, boomer(60, "sweperf shutdown started")]) + "\n") + trial = {"timestamp": "2026-10-11T07:06:40Z", "tag": TAG, "test_name": d.name, "metric": "trial_summary", + "measurements": {"node_count": "1", "allocatable_cores": "100", "allocatable_ram_gb": "400", + "aggregate_failure_ratio": "0.0005", "metadata_cluster_name": cluster}} + (d / "stats.jsonl").write_text(json.dumps(trial) + "\n") + (d / "server_summary.json").write_text(json.dumps({"metadata": { + "start_ts": T0, "end_ts": T0 + 60, "steady_start_ts": T0, "steady_end_ts": T0 + 60}})) + return d + + +def stored(root: Path, test: str, run_ts: int, records=None, cluster: str = "c1") -> Path: + date = datetime.fromtimestamp(run_ts, tz=timezone.utc).strftime("%Y-%m-%d") + d = Path(rundir.run_uri(str(root), test, date, str(run_ts), TAG)) + return write_run(d, records or [record(10, "a1"), record(20, "a2", wall_ms=9000.0)], cluster=cluster) + + +def evaluations(limit: float = 90) -> list[dict]: + """goodput_strict against the run of t1 (the 1-user run), and an envelope check.""" + rec_sel = "has(line.msg) && line.msg == 'sweperf task record' && has(line.v) && line.v == 1.0" + goodput_cfg = { + "inputs": [ + {"name": "run", "reader": "json_lines", "reader_cfg": {"file": "logs.txt", "prefix": "[boomer] ", "select": rec_sel}}, + {"name": "one_user", "from_test": "t1", "reader": "json_lines", + "reader_cfg": {"file": "logs.txt", "prefix": "[boomer] ", "select": rec_sel}}, + ], + "locals": [ + {"name": "ref_wall_ms", "cel": "percentile(one_user.lines.filter(t, !t.failed).map(t, t.wall_ms), 50.0)"}, + {"name": "slow_wall", "cel": "run.lines.filter(t, t.wall_ms > 1.5 * ref_wall_ms)"}, + {"name": "good", + "cel": "run.lines.filter(t, !t.failed && t.resume_max_ms <= 1500.0 && t.wall_ms <= 1.5 * ref_wall_ms)"}, + ], + "values": [ + {"name": "good_ratio_pct", "cel": "round(100.0 * double(size(good)) / double(size(run.lines)), 2)"}, + {"name": "good", "cel": "double(size(good))"}, + {"name": "slow_wall", "cel": "double(size(slow_wall))"}, + {"name": "one_user_run", "cel": "one_user.uri"}, + ]} + return [ + {"name": "goodput_strict", "variables": [{"name": "goodput", "source": "cel", "source_cfg": goodput_cfg}], + "conditions": [{"name": "good_ratio", "cel": f"goodput.good_ratio_pct >= {limit}"}]}, + {"name": "envelope", + "variables": [{"name": "trial", "source": "run_results", + "source_cfg": {"metric": "trial_summary", "measurements": ["aggregate_failure_ratio"]}}], + "conditions": [{"name": "failure_ratio", "cel": "trial.aggregate_failure_ratio <= 0.001"}]}, + ] + + +def goodput(out: Path) -> dict: + """The goodput variable of the evaluation.json in out.""" + return json.loads((out / "evaluation.json").read_text())["evaluations"][0]["variables"][0] + + +class CliTest(unittest.TestCase): + def run_cli(self, *argv: str) -> tuple[int, str, str]: + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + rc = cli.main(list(argv)) + return rc, out.getvalue(), err.getvalue() + + def test_evaluates_and_reevaluates_from_run(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + t1 = stored(root, "t1", T0) + run = stored(root, "t150", T0 + 600, records=[record(10, "a1"), record(20, "a2", wall_ms=13000.0)]) + evs = root / "evs.json" + evs.write_text(json.dumps(evaluations(limit=75))) + out = root / "out" + rc, printed, _ = self.run_cli(f"{run}/", "--evaluations", f"@{evs}", "--out", str(out)) + self.assertEqual(rc, 0) + rows = [json.loads(line) for line in (out / "evaluation_rows.jsonl").read_text().splitlines()] + self.assertEqual({r["timestamp"] for r in rows}, {"2026-10-11T07:06:40Z"}) # from the run's stats.jsonl + self.assertEqual(len(rows), 3) + # t1's run is the 1-user run: a2's 13 s is over 1.5 x its 8.5 s median. + self.assertIn("evaluation goodput_strict: fail (failed: good_ratio)", printed) + values = goodput(out)["values"] + self.assertEqual((values["one_user_run"], values["slow_wall"]), (str(t1), 1.0)) + report = json.loads((out / "evaluation.json").read_text()) + self.assertEqual((report["run"]["test_name"], report["run"]["run_ts"]), ("t150", str(T0 + 600))) + self.assertFalse((run / "evaluation.json").exists()) # never writes into the stored run + # The stored run's evaluation.json (as the runner uploads it) re-evaluates as recorded. + (run / "evaluation.json").write_text((out / "evaluation.json").read_text()) + rc, printed, _ = self.run_cli(str(run), "--from-run", "--out", str(root / "o2")) + self.assertEqual(rc, 0) + self.assertEqual(json.loads((root / "o2" / "evaluation.json").read_text())["summary"], report["summary"]) + + def test_paired_runs_need_the_runners_layout(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + t1 = stored(root, "t1", T0) + flat = write_run(root / "flat", [record(10, "a1")]) + rc, _, _ = self.run_cli(str(flat), "--evaluations", json.dumps(evaluations()), "--out", str(root / "o1")) + self.assertEqual(rc, 0) + self.assertEqual(goodput(root / "o1")["invalid_reason"], "input 'one_user': no run of t1") + # A symlink into a runs/ tree is stored there: its path gives the identity and the dest. + link = Path(rundir.run_uri(tmp, "t150", "2026-10-11", str(T0 + 600), TAG)) + link.parent.mkdir(parents=True) + link.symlink_to(flat) + rc, _, _ = self.run_cli(str(link), "--evaluations", json.dumps(evaluations()), "--out", str(root / "o2")) + self.assertEqual(rc, 0) + report = json.loads((root / "o2" / "evaluation.json").read_text()) + self.assertEqual(report["run"]["run_ts"], str(T0 + 600)) + self.assertEqual(goodput(root / "o2")["values"]["one_user_run"], str(t1)) + + def test_gcs_run(self): + """A gs:// RUN is read, and its paired run found and read, under the same gs:// dest.""" + with tempfile.TemporaryDirectory() as tmp: + bucket = Path(tmp) / "bucket" + + def gcs_list(prefix: str) -> list[str]: + local = bucket / prefix[len("gs://b/"):] + return [f"gs://b/{p.relative_to(bucket)}" for p in local.rglob("*") if p.is_file()] + + def gcs_download(uri: str, dest: Path) -> Optional[Path]: + src = bucket / uri[len("gs://b/"):] + if not src.is_file(): + return None + dest.parent.mkdir(parents=True, exist_ok=True) + return Path(shutil.copyfile(src, dest)) + + t1 = stored(bucket / "p", "t1", T0) + run = stored(bucket / "p", "t150", T0 + 600) + uri = f"gs://b/{run.relative_to(bucket)}" + with mock.patch.object(rundir, "_gcs_list", side_effect=gcs_list), \ + mock.patch.object(rundir, "_gcs_download", side_effect=gcs_download): + rc, _, _ = self.run_cli(uri, "--evaluations", json.dumps(evaluations()), "--out", f"{tmp}/out") + self.assertEqual(rc, 0) + values = goodput(Path(tmp) / "out")["values"] + self.assertEqual((values["one_user_run"], values["good"]), (f"gs://b/{t1.relative_to(bucket)}", 2.0)) + + def test_exit_codes(self): + with tempfile.TemporaryDirectory() as tmp: + run = write_run(Path(tmp) / "run", [record(10, "a1")]) + self.assertEqual(self.run_cli(str(run), "--evaluations", '[{"name": 1}]')[0], 2) + self.assertEqual(self.run_cli(str(run), "--from-run")[0], 2) # no evaluation.json in the run + self.assertEqual(self.run_cli(str(Path(tmp) / "missing"), "--evaluations", + json.dumps(evaluations()))[0], 1) + # A shape no check expects is a config error rather than a traceback. + bad_cel = evaluations() + bad_cel[0]["conditions"][0]["cel"] = ["<="] + self.assertEqual(self.run_cli("--check-config", "--evaluations", json.dumps(bad_cel))[0], 2) + self.assertEqual(self.run_cli("--check-config", "--evaluations", json.dumps(evaluations()))[0], 0) + with self.assertRaises(SystemExit): + self.run_cli(str(run)) # no evaluations given + + def test_check_config_on_a_tests_file(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "tests.yaml" + path.write_text(json.dumps({"tests": [{"name": "ok", "evaluations": evaluations()}, + {"name": "bad", "evaluations": [{"name": "Bad"}]}, + {"name": "none"}]})) + rc, out, err = self.run_cli("--check-config", "--tests-file", str(path)) + self.assertEqual(rc, 2) + self.assertIn("ok: ok (2 evaluations)", out) + self.assertIn("bad: evaluations: 0.name: String should match pattern", err) + rc, out, _ = self.run_cli("--check-config", "--tests-file", str(path), "--test-name", "ok") + self.assertEqual((rc, out), (0, "ok: ok (2 evaluations)\n")) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarking/evaluate/unit_tests/test_evaluator.py b/benchmarking/evaluate/unit_tests/test_evaluator.py index 4bd854fda1..71be96a778 100644 --- a/benchmarking/evaluate/unit_tests/test_evaluator.py +++ b/benchmarking/evaluate/unit_tests/test_evaluator.py @@ -115,7 +115,7 @@ def test_load_and_echo(self): echoed = config.to_json(evs) self.assertEqual(config.to_json(config.load(echoed)), echoed) self.assertEqual(echoed[0]["description"], "") - # Config names allow double underscores; external keys allow capitals and dots. + # Evaluation and condition names allow double underscores; external keys allow capitals and dots. dotted = [{ "name": "ingress__rps", "variables": [{"name": "nh", "source": "run_results", "source_cfg": { @@ -170,6 +170,10 @@ def broken(path: str, value) -> list: "0: condition names repeated": broken("conditions", evaluations()[0]["conditions"] * 2), "0: variable names repeated": broken("variables.1.name", "trial"), "evaluations: evaluation names repeated": evaluations() + evaluations(), + # trial__x with value y and trial with value x__y would both be trial__x__y. + "0.variables.0.name: String should match pattern '^[a-z][a-z0-9]*(_[a-z0-9]+)*$'": broken( + "variables.0.name", "trial__x"), + "0.variables.1.name: String should match pattern": broken("variables.1.name", "server_"), } for message, evs in cases.items(): with self.assertRaisesRegex(config.ConfigError, re.escape(message), msg=message): diff --git a/benchmarking/evaluate/unit_tests/test_outputs.py b/benchmarking/evaluate/unit_tests/test_outputs.py new file mode 100644 index 0000000000..badcd2fd78 --- /dev/null +++ b/benchmarking/evaluate/unit_tests/test_outputs.py @@ -0,0 +1,136 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the outputs: evaluation.json, the stats.jsonl rows and the log lines.""" + +import json +import sys +import tempfile +import unittest +from dataclasses import replace +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from evaluate import outputs # noqa: E402 +from evaluate.evaluator import ( # noqa: E402 + ConditionResult, EvaluationResult, Report, Summary, VariableResult, error_report) +from evaluate.sources.base import RunIdentity # noqa: E402 + +TAG = "abc123" +WORKER_MEMORY = "server.working_set_node_max_gb < 0.85 * trial.allocatable_ram_gb" +FAILURE_RATE = ConditionResult(name="failure_rate", cel="trial.aggregate_failure_ratio <= 0.001", status="pass", + passed=True, reason="") +ENVELOPE = EvaluationResult( + name="envelope_health", description="", status="fail", passed=False, reason="failed: worker_memory", + conditions=(FAILURE_RATE, ConditionResult(name="worker_memory", cel=WORKER_MEMORY, status="fail", passed=False, + reason=f"worker_memory: {WORKER_MEMORY}")), + variables=(VariableResult(name="trial", source="run_results", status="ok", invalid_reason=None, + values={"aggregate_failure_ratio": 0.0005, "allocatable_ram_gb": 400.0}), + VariableResult(name="server", source="run_results", status="ok", invalid_reason=None, + values={"working_set_node_max_gb": 350.0})), + config={"name": "envelope_health"}) + + +def report(ev: EvaluationResult, summary: Summary) -> Report: + return Report(run={"test_name": "t150", "tag": TAG}, evaluated_at="2026-11-02T10:00:00Z", error=None, + summary=summary, evaluations=(ev,)) + + +FAILED = report(ENVELOPE, Summary(status="fail", passed=False, failed=("envelope_health",), invalid=())) + + +class OutputsTest(unittest.TestCase): + def test_rows(self): + rows = outputs.stats_rows(FAILED, "ts", TAG, "t150") + self.assertEqual([r["metric"] for r in rows], ["evaluation_envelope_health", "evaluations"]) + self.assertTrue(all(set(r) == {"timestamp", "tag", "test_name", "metric", "measurements"} for r in rows)) + self.assertEqual(rows[0]["measurements"], { + "status": "fail", "passed": "false", "reason": "failed: worker_memory", + "failed_conditions": "worker_memory", "invalid_conditions": "", + "trial__aggregate_failure_ratio": "0.0005", "trial__allocatable_ram_gb": "400", + "server__working_set_node_max_gb": "350"}) + self.assertEqual(rows[1]["measurements"], {"status": "fail", "passed": "false", "failed": "envelope_health", + "invalid": "", "error": ""}) + + def test_value_names_are_kept_and_nulls_stay_null(self): + nh = VariableResult(name="nh", source="run_results", status="ok", invalid_reason=None, + values={"metric_nighthawk.builtin_attempted_rps": 600.0, "p99.9": None, + "release": "2026_11"}) + passed = replace(ENVELOPE, status="pass", passed=True, reason="", conditions=(FAILURE_RATE,), variables=(nh,)) + m = outputs.stats_rows(report(passed, Summary(status="pass", passed=True, failed=(), invalid=())), + "ts", TAG, "t")[0]["measurements"] + self.assertEqual({k: v for k, v in m.items() if k.startswith("nh__")}, { + "nh__metric_nighthawk.builtin_attempted_rps": "600", "nh__p99.9": None, "nh__release": "2026_11"}) + + def test_summary_lines(self): + self.assertEqual(outputs.summary_lines(FAILED), [ + f"evaluation envelope_health: fail (failed: worker_memory)\n worker_memory: {WORKER_MEMORY}", + "summary: fail (1 evaluations)", + ]) + why = "server is invalid: no stats.jsonl row server_summary" + invalid = replace(ENVELOPE, status="invalid", reason="invalid: worker_memory, headroom", conditions=( + FAILURE_RATE, + ConditionResult(name="worker_memory", cel=WORKER_MEMORY, status="invalid", passed=False, reason=why), + ConditionResult(name="headroom", cel="server.working_set_node_max_gb < 100", status="invalid", + passed=False, reason=why))) + summary = Summary(status="invalid", passed=False, failed=(), invalid=("envelope_health",)) + self.assertEqual(outputs.summary_lines(report(invalid, summary)), [ + f"evaluation envelope_health: invalid (invalid: worker_memory, headroom)\n {why}", + "summary: invalid (1 evaluations)", + ]) + passed = replace(ENVELOPE, status="pass", passed=True, reason="", conditions=(FAILURE_RATE,)) + self.assertEqual(outputs.summary_lines(report(passed, Summary(status="pass", passed=True, failed=(), + invalid=()))), [ + "evaluation envelope_health: pass", + "summary: pass (1 evaluations)", + ]) + + def test_error_report(self): + failed_to_load = error_report(RunIdentity(uri="/runs/t150", test_name="t150"), + "config: evaluations: 0.name: String should match pattern") + self.assertEqual(failed_to_load.run, {"test_name": "t150", "uri": "/runs/t150"}) + rows = outputs.stats_rows(failed_to_load, "ts", TAG, "t150") + self.assertEqual([(r["metric"], r["measurements"]) for r in rows], [("evaluations", { + "status": "invalid", "passed": "false", "failed": "", "invalid": "", + "error": "config: evaluations: 0.name: String should match pattern"})]) + self.assertEqual(outputs.summary_lines(failed_to_load), + ["evaluations not run: config: evaluations: 0.name: String should match pattern"]) + + def test_evaluation_json(self): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "run" / "evaluation.json" + outputs.write_evaluation_json(FAILED, out) + doc = json.loads(out.read_text()) + # The report is serialized before anything is written. + bad = Path(tmp) / "bad.json" + with self.assertRaises(TypeError): + outputs.write_evaluation_json(replace(FAILED, run={"bad": object()}), bad) + self.assertFalse(bad.exists()) + self.assertEqual(list(doc), ["run", "evaluated_at", "error", "summary", "evaluations"]) + self.assertEqual(doc["summary"], {"status": "fail", "passed": False, "failed": ["envelope_health"], + "invalid": []}) + (ev,) = doc["evaluations"] + self.assertEqual(list(ev), ["name", "description", "status", "passed", "reason", "conditions", "variables", + "config"]) + self.assertEqual(list(ev["conditions"][0].items()), [ + ("name", "failure_rate"), ("cel", "trial.aggregate_failure_ratio <= 0.001"), ("status", "pass"), + ("passed", True), ("reason", "")]) + self.assertEqual(list(ev["variables"][1].items()), [ + ("name", "server"), ("source", "run_results"), ("status", "ok"), ("invalid_reason", None), + ("values", {"working_set_node_max_gb": 350.0})]) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarking/evaluate/unit_tests/test_rundir.py b/benchmarking/evaluate/unit_tests/test_rundir.py new file mode 100644 index 0000000000..d52a09017f --- /dev/null +++ b/benchmarking/evaluate/unit_tests/test_rundir.py @@ -0,0 +1,206 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for rundir: stored runs, and finding the run another run is paired with. + +Runs are stand-in directories in the runner's layout; GCS is mocked. +""" + +import json +import sys +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from evaluate import rundir # noqa: E402 +from evaluate.sources.base import LocalArtifacts, RunIdentity # noqa: E402 + +T0 = 1791700000 # 2026-10-11T07:06:40Z +TAG = "abc123" + + +def write_run(d: Path, cluster: str = "c1") -> Path: + d.mkdir(parents=True, exist_ok=True) + (d / "logs.txt").write_text("[boomer] {}\n") + trial = {"timestamp": "2026-10-11T07:06:40Z", "tag": TAG, "test_name": "t", "metric": "trial_summary", + "measurements": {"metadata_cluster_name": cluster} if cluster else {}} + (d / "stats.jsonl").write_text(json.dumps(trial) + "\n") + return d + + +def stored(root: Path, test: str, run_ts: int, tag: str = TAG, cluster: str = "c1") -> Path: + date = datetime.fromtimestamp(run_ts, tz=timezone.utc).strftime("%Y-%m-%d") + return write_run(Path(rundir.run_uri(str(root), test, date, str(run_ts), tag)), cluster) + + +class FindPairedRunTest(unittest.TestCase): + def test_latest_run_with_the_tag_before_the_run(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + want = stored(root, "t1", T0 + 100) + stored(root, "t1", T0 + 50) # older + stored(root, "t1", T0 + 200) # after the run + stored(root, "t1", T0 + 120, tag="other") # another commit, e.g. a fork job reusing the name + stored(root, "t1", T0 - 8 * 3600) # too old + self.assertEqual(rundir.find_paired_run(str(root), "t1", TAG, T0 + 150), str(want)) + with self.assertRaisesRegex(LookupError, f"^no run of t1 with tag abc123 under {tmp}/runs/ in the " + f"6 h before run_ts {T0 + 10}$"): + rundir.find_paired_run(str(root), "t1", TAG, T0 + 10) + + def test_pairing_crosses_midnight(self): + midnight = 1791763200 # 2026-10-12T00:00:00Z + with tempfile.TemporaryDirectory() as tmp: + want = stored(Path(tmp), "t1", midnight - 600) + self.assertEqual(rundir.find_paired_run(tmp, "t1", TAG, midnight + 600), str(want)) + + def test_gcs_dest(self): + prefix = "gs://b/p/runs/t1/run_date=2026-10-11" + objects = [f"{prefix}/run_ts={T0 + 100}/run_tag={TAG}/logs.txt", + f"{prefix}/run_ts={T0 + 100}/run_tag={TAG}/stats.jsonl", + f"{prefix}/run_ts={T0 + 50}/run_tag={TAG}/stats.jsonl", + f"{prefix}/run_ts={T0 + 120}/run_tag={TAG}/status.json", # no logs.txt or stats.jsonl + f"{prefix}/run_ts={T0 + 130}/run_tag=other/logs.txt"] + with mock.patch.object(rundir, "_gcs_list", return_value=objects) as listing: + found = rundir.find_paired_run("gs://b/p/", "t1", TAG, T0 + 150) + self.assertEqual(found, f"{prefix}/run_ts={T0 + 100}/run_tag={TAG}") + listing.assert_called_once_with(f"{prefix}/") + + def test_identity_from_uri(self): + self.assertEqual(rundir.identity_from_uri(f"gs://b/p/runs/t1/run_date=2026-10-11/run_ts={T0}/run_tag=x/"), + RunIdentity(uri=f"gs://b/p/runs/t1/run_date=2026-10-11/run_ts={T0}/run_tag=x", + test_name="t1", tag="x", run_date="2026-10-11", run_ts=T0)) + self.assertEqual(rundir.identity_from_uri("/tmp/run"), RunIdentity(uri="/tmp/run")) + + +class OpenRunTest(unittest.TestCase): + def test_local_run(self): + with tempfile.TemporaryDirectory() as tmp: + d = write_run(Path(tmp) / "run") + run = rundir.open_run(f"{d}/", Path(tmp) / "x") + self.assertEqual(run.identity, RunIdentity(uri=str(d))) + self.assertEqual(run.artifact("logs.txt"), d / "logs.txt") + self.assertIsNone(run.artifact("server_summary.json")) + with self.assertRaisesRegex(FileNotFoundError, "no run directory"): + rundir.open_run(str(Path(tmp) / "missing"), Path(tmp) / "x") + + def test_gcs_run_downloads_each_artifact_on_first_use(self): + uri = f"gs://b/p/runs/t1/run_date=2026-10-11/run_ts={T0}/run_tag={TAG}" + path = uri[len("gs://b/"):] + stored_names = {f"{path}/logs.txt", f"{path}/custom/extra.json"} + downloads = [] + + class Blob: + def __init__(self, name): + self.name = name + + def download_to_filename(self, dest): + Path(dest).write_text(self.name) + + def get_blob(name): + downloads.append(name) + return Blob(name) if name in stored_names else None + + client = mock.Mock() + client.list_blobs.return_value = [Blob(n) for n in stored_names] + client.bucket.return_value.get_blob.side_effect = get_blob + with tempfile.TemporaryDirectory() as tmp: + with mock.patch.object(rundir, "_storage_client", return_value=client): + run = rundir.open_run(uri, Path(tmp)) + self.assertEqual(run.identity, rundir.identity_from_uri(uri)) + self.assertEqual(downloads, []) # nothing is read until a source asks + # A file the framework knows nothing about is there too, for the source that reads it. + self.assertEqual(run.artifact("custom/extra.json").read_text(), f"{path}/custom/extra.json") + self.assertEqual(run.artifact("logs.txt").read_text(), f"{path}/logs.txt") + self.assertEqual(run.artifact("logs.txt").parent.parent, Path(tmp)) + self.assertIsNone(run.artifact("server_summary.json")) + self.assertIsNone(run.artifact("server_summary.json")) + with self.assertRaisesRegex(ValueError, "outside the run"): + run.artifact("../escape.txt") + self.assertEqual(downloads, [f"{path}/custom/extra.json", f"{path}/logs.txt", + f"{path}/server_summary.json"]) # each once + client.list_blobs.return_value = [] + with mock.patch.object(rundir, "_storage_client", return_value=client), \ + self.assertRaisesRegex(FileNotFoundError, f"no objects under {uri}/"): + rundir.open_run(uri, Path(tmp)) + + +class PairedRunFinderTest(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + self.run_dir = stored(self.root, "t150", T0 + 3600) + + def finder(self, run_dir=None): + return rundir.paired_run_finder(str(self.root), TAG, T0 + 3600, LocalArtifacts(run_dir or self.run_dir), + self.root / "tmp") + + def test_finds_the_paired_run(self): + want = stored(self.root, "t1", T0) + paired = self.finder()("t1") + self.assertEqual(paired.identity, rundir.identity_from_uri(str(want))) + self.assertEqual(paired.artifact("stats.jsonl"), want / "stats.jsonl") + + def test_lookup_errors(self): + stored(self.root, "t1", T0, cluster="sweperf-load-test") + stored(self.root, "t2", T0, cluster="") + stored(self.root, "t3", T0, cluster="c1") + finder = self.finder() + with self.assertRaisesRegex(LookupError, "^cluster differs \\(c1 vs sweperf-load-test\\)$"): + finder("t1") + self.assertEqual(finder("t2").identity.test_name, "t2") # no cluster recorded + self.assertEqual(finder("t3").identity.test_name, "t3") + with self.assertRaisesRegex(LookupError, "^no run of t4 with tag abc123 under "): + finder("t4") + # Anything else is a LookupError too, saying what it was. + for error, why in ((PermissionError("403 list"), "PermissionError: 403 list"), + (KeyError("run_ts"), "KeyError: 'run_ts'")): + with self.subTest(why=why), mock.patch.object(rundir, "find_paired_run", side_effect=error), \ + self.assertRaises(LookupError) as raised: + self.finder()("t1") + self.assertEqual((type(raised.exception), str(raised.exception)), (LookupError, why)) + + def test_a_corrupt_stats_jsonl_is_a_lookup_error(self): + stored(self.root, "t1", T0) + with open(self.run_dir / "stats.jsonl", "a", encoding="utf-8") as f: + f.write("not json\n") + with self.assertRaisesRegex(LookupError, "^can't compare the clusters: stats.jsonl line 2 isn't JSON"): + self.finder()("t1") + + def test_each_test_is_looked_up_once(self): + stored(self.root, "t1", T0) + finder = self.finder() + with mock.patch.object(rundir, "find_paired_run", wraps=rundir.find_paired_run) as find: + self.assertIs(finder("t1"), finder("t1")) + for _ in range(2): + with self.assertRaisesRegex(LookupError, "^no run of t9 "): + finder("t9") + self.assertEqual([c.args[1] for c in find.call_args_list], ["t1", "t9"]) + + def test_cluster_is_read_when_asked(self): + """The runner makes the finder before its run writes stats.jsonl.""" + stored(self.root, "t1", T0, cluster="c1") + later = self.root / "work" + finder = self.finder(later) + write_run(later, cluster="c2") + with self.assertRaisesRegex(LookupError, "^cluster differs \\(c2 vs c1\\)$"): + finder("t1") + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarking/evaluate/unit_tests/test_test_entries.py b/benchmarking/evaluate/unit_tests/test_test_entries.py new file mode 100644 index 0000000000..b6d942fcc2 --- /dev/null +++ b/benchmarking/evaluate/unit_tests/test_test_entries.py @@ -0,0 +1,67 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Every evaluations block checked into the repository loads: the tests*.yaml +entries the orchestrator runs (its image can only check their shape), and the +examples in the docs.""" + +import re +import sys +import unittest +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(ROOT / "benchmarking")) + +from evaluate import config # noqa: E402 + + +class TestEntriesTest(unittest.TestCase): + def test_tests_yaml_evaluations_load(self): + files = sorted((ROOT / "benchmarking" / "automation").glob("tests*.yaml")) + self.assertTrue(files) + for path in files: + for test in yaml.safe_load(path.read_text())["tests"]: + if "evaluations" in test: + with self.subTest(file=path.name, test=test["name"]): + config.load(test["evaluations"]) + + def test_documented_examples_load(self): + docs = [ROOT / "benchmarking" / "evaluate" / "README.md", ROOT / "benchmarking" / "automation" / "tests.yaml"] + blocks = [] + for path in docs: + text = path.read_text() + if path.suffix == ".yaml": # the commented example in the header + lines = text.splitlines() + start = lines.index("# evaluations:") + example = [] + for line in lines[start:]: + if not line.startswith("# "): + break + example.append(line[4:]) + text = "```yaml\n" + "\n".join(example) + "\n```" + for block in re.findall(r"```yaml\n(.*?)```", text, re.S): + doc = yaml.safe_load(block) + entries = doc if isinstance(doc, list) else [doc] + blocks += [(path.name, e["evaluations"]) for e in entries if isinstance(e, dict) and "evaluations" in e] + self.assertGreaterEqual(len(blocks), 3) + for name, block in blocks: + with self.subTest(doc=name): + config.load(block) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarking/locust/Dockerfile b/benchmarking/locust/Dockerfile index 8fb8f61a6a..1a62248355 100644 --- a/benchmarking/locust/Dockerfile +++ b/benchmarking/locust/Dockerfile @@ -63,6 +63,10 @@ COPY benchmarking/locust/phase_breakdown.py /app/phase_breakdown.py COPY benchmarking/locust/ladder_steps.py /app/ladder_steps.py COPY benchmarking/locust/turnover_window.py /app/turnover_window.py COPY benchmarking/analysis/phase_report.py /app/phase_report.py +# Run evaluations (the --test-entry's evaluations), imported by runner.py as the evaluate package. +# The modules only: the directory also holds the tests and a local venv. +COPY benchmarking/evaluate/*.py /app/evaluate/ +COPY benchmarking/evaluate/sources/ /app/evaluate/sources/ ENV PYTHONPATH=/app:/app/deps ENV PYTHONUNBUFFERED=1 diff --git a/benchmarking/locust/requirements.txt b/benchmarking/locust/requirements.txt index a76e292a41..9c35c266e5 100644 --- a/benchmarking/locust/requirements.txt +++ b/benchmarking/locust/requirements.txt @@ -28,3 +28,6 @@ opentelemetry-instrumentation-grpc opentelemetry-instrumentation-requests google-cloud-storage kubernetes +# Run evaluations (benchmarking/evaluate/requirements.txt). +cel-expr-python==0.1.3 +pydantic>=2 diff --git a/benchmarking/locust/runner.py b/benchmarking/locust/runner.py index 58b215b56a..cc0355e67c 100644 --- a/benchmarking/locust/runner.py +++ b/benchmarking/locust/runner.py @@ -194,6 +194,18 @@ def parse_args() -> argparse.Namespace: "and a 10s scrape" ), ) + p.add_argument( + "--test-entry", + type=json_object, + default={}, + help=( + "The run's test entry from tests.yaml, as a JSON object. The runner " + "reads its `evaluations` (see benchmarking/evaluate/README.md): " + "their sources are set up before the run; after it they judge the " + "run, and the verdicts are written to evaluation.json, plus rows " + "in stats.jsonl. They never change the exit code or status.json" + ), + ) p.add_argument( "--phase-breakdown", action=argparse.BooleanOptionalAction, @@ -210,6 +222,17 @@ def parse_args() -> argparse.Namespace: return args +def json_object(value: str) -> dict[str, Any]: + """An argparse type: a JSON object.""" + try: + obj = json.loads(value) + except ValueError as e: + raise argparse.ArgumentTypeError(f"not JSON: {e}") from None + if not isinstance(obj, dict): + raise argparse.ArgumentTypeError("not a JSON object") + return obj + + # Tests whose User class is implemented in Python. Closed set: new load # generators are written in boomer, so anything else runs on boomer. Remove # entries as these are retired; when empty, drop needs_boomer entirely. @@ -288,6 +311,7 @@ def log_run_config(args: argparse.Namespace, dest_prefix: str, work_dir: Path, l f" users: {args.users}", f" uses_boomer: {needs_boomer(args.file)}", f" empty stats ok: {args.allow_empty_stats}", + f" evaluations: {evaluation_names(args.test_entry.get('evaluations'))}", f" dest_prefix: {dest_prefix}", f" work_dir: {work_dir}", # One flag for each line, with its value. A run gives many flags to @@ -301,6 +325,90 @@ def log_run_config(args: argparse.Namespace, dest_prefix: str, work_dir: Path, l tee(logs, line) +def evaluation_names(evaluations: Any) -> str: + """The names in a test entry's evaluations, for the run config; never raises.""" + if evaluations is None: + return "(none)" + try: + return ", ".join(str(e.get("name")) for e in evaluations) or "(empty)" + except (TypeError, AttributeError): + return "(unreadable, see below)" + + +def prepare_evaluations( + args: argparse.Namespace, run_dir: Path, identity: Any, logs: TextIO +) -> tuple[Any, str | None]: + """The evaluator of the test entry's evaluations, whose sources start + with the run, or None and why it can't judge the run. Called before the + run so a typo is in the log from the start; the run goes ahead either + way. Sources ask for paired runs, such as the suite's 1-user run, under + --dest with the same --tag. run_dir holds the run's artifacts under the + names they are stored under (see link_artifacts); identity (an + evaluate.RunIdentity) says which run it is and where it is stored.""" + evaluations = args.test_entry.get("evaluations") + if evaluations is None: + return None, None + try: + import evaluate # benchmarking/evaluate, at /app/evaluate in the image + from evaluate import rundir + + error = evaluate.check_config(evaluations) + if error is None: + artifacts = evaluate.LocalArtifacts(run_dir) + run_of = rundir.paired_run_finder( + args.dest, args.tag, identity.run_ts, artifacts, run_dir.parent / "evaluate" + ) + run = evaluate.BenchmarkRunContext(identity, artifacts, run_of=run_of) + return evaluate.prepare(evaluations, run), None + except Exception as e: # never cost the run its load test + error = f"evaluate failed to load or prepare the evaluations: {type(e).__name__}: {e}" + tee(logs, f"Warning: evaluations will not be judged: {error}") + return None, error + + +def run_identity(args: argparse.Namespace, run_date: str, run_ts: int, uri: str) -> Any: + """The evaluate.RunIdentity of the run stored at uri, or None when + evaluate can't be imported (the evaluations then say why).""" + try: + import evaluate + except Exception: # never cost the run its load test + return None + return evaluate.RunIdentity( + uri=uri, test_name=args.name, tag=args.tag, run_date=run_date, run_ts=run_ts + ) + + +def link_artifacts(files: list[tuple[Path, str]], run_dir: Path) -> None: + """Links each stored name in run_dir to its file, which need not exist yet, + so the evaluations' sources read the run as it will be stored.""" + run_dir.mkdir(parents=True, exist_ok=True) + for src, basename in files: + link = run_dir / basename + link.unlink(missing_ok=True) # a rerun in the same second reuses the work dir + link.symlink_to(src.absolute()) + + +def run_evaluations( + args: argparse.Namespace, + evaluator: Any, + error: str | None, + identity: Any, + out_json: Path, + jsonl_path: Path, + data_ts: str, + logs: TextIO, +) -> None: + """Judges the finished run with the evaluator from prepare_evaluations: + evaluation.json and stats.jsonl rows. Without one, evaluation.json + records the error, under identity (an evaluate.RunIdentity), instead.""" + import evaluate + + report = evaluate.error_report(identity, error) if evaluator is None else evaluator.evaluate() + evaluate.write(report, out_json, jsonl_path, data_ts, args.tag, args.name) + for line in evaluate.summary_lines(report): + tee(logs, line) + + def extract_trace_record(prefix: str, line: str) -> dict[str, str] | None: """Parse `line` into a trace record dict (TRACE_COLUMNS keys) if it describes a sampled span, else return None. Handles boomer slog JSON @@ -569,16 +677,36 @@ def main() -> None: traces_path = work_dir / f"{args.name}_traces.txt" status_path = work_dir / f"{args.name}_status.json" server_summary_json = work_dir / f"{args.name}_server_summary.json" + stats_history_csv = work_dir / f"{args.name}_stats_history.csv" + evaluation_json = work_dir / f"{args.name}_evaluation.json" prefix = ( f"{args.dest.rstrip('/')}/runs/{args.name}" f"/run_date={run_date}/run_ts={run_ts}/run_tag={args.tag}" ) + # The run's artifacts and the names they are stored under. + files: list[tuple[Path, str]] = [ + (status_path, "status.json"), + (logs_path, "logs.txt"), + (traces_path, "traces.txt"), + (jsonl_path, "stats.jsonl"), + (stats_csv, "stats.csv"), + (work_dir / f"{args.name}_exceptions.csv", "exceptions.csv"), + (work_dir / f"{args.name}_failures.csv", "failures.csv"), + (work_dir / f"{args.name}_stats_history.csv", "stats_history.csv"), + (server_summary_json, "server_summary.json"), + (evaluation_json, "evaluation.json"), + ] + run_dir = work_dir / "run" + link_artifacts(files, run_dir) + identity = run_identity(args, run_date, run_ts, prefix) + with open(logs_path, "w") as logs, open(traces_path, "w") as traces: traces.write("\t".join(TRACE_COLUMNS) + "\n") traces.flush() log_run_config(args, prefix, work_dir, logs) + evaluator, evaluations_error = prepare_evaluations(args, run_dir, identity, logs) # Worker pods around the run, for OOMKilled restarts and evictions. pods_before = read_worker_pods(args, logs) exit_code = run_test(args, csv_prefix, logs, traces) @@ -613,7 +741,6 @@ def main() -> None: # Density frontiers and server-side telemetry are additive. They are # kept out of the block above so that a failure here cannot discard # the measurements the trial actually came for. - stats_history_csv = work_dir / f"{args.name}_stats_history.csv" # Seeded up front so that a discovery failure still leaves a usable # value for the frontier summary below. facts = dict(EMPTY_FACTS) @@ -716,23 +843,32 @@ def main() -> None: elif stats_generated: tee(logs, "Skipping phase breakdown (--no-phase-breakdown)") + # Evaluations come last: their sources read the finished logs.txt and + # stats.jsonl, so they run once logs.txt is closed, and append their own + # lines to it. Additive like the steps above: a failure here costs no + # measurement. + if args.test_entry.get("evaluations") is not None: + with open(logs_path, "a") as logs: + try: + run_evaluations( + args, + evaluator, + evaluations_error, + identity=identity, + out_json=evaluation_json, + jsonl_path=jsonl_path, + data_ts=data_ts, + logs=logs, + ) + except Exception as e: + tee(logs, f"Warning: Failed to run the evaluations: {e}") + status_path.write_text( json.dumps( {"locust_exit_code": exit_code, "stats_generated": stats_generated} ) ) - files: list[tuple[Path, str]] = [ - (status_path, "status.json"), - (logs_path, "logs.txt"), - (traces_path, "traces.txt"), - (jsonl_path, "stats.jsonl"), - (stats_csv, "stats.csv"), - (work_dir / f"{args.name}_exceptions.csv", "exceptions.csv"), - (work_dir / f"{args.name}_failures.csv", "failures.csv"), - (work_dir / f"{args.name}_stats_history.csv", "stats_history.csv"), - (server_summary_json, "server_summary.json"), - ] for src, basename in files: if not src.exists(): print(f"Skipping {src}: not produced", flush=True) diff --git a/benchmarking/locust/unit_tests/test_runner_evaluations.py b/benchmarking/locust/unit_tests/test_runner_evaluations.py new file mode 100644 index 0000000000..42adc38f1b --- /dev/null +++ b/benchmarking/locust/unit_tests/test_runner_evaluations.py @@ -0,0 +1,153 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for runner.py's evaluations (the --test-entry's evaluations).""" + +import argparse +import io +import json +import sys +import tempfile +import unittest +import unittest.mock +from datetime import datetime, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +# The evaluate package is /app/evaluate in the image; here it is +# benchmarking/evaluate. Appended, so nothing else on the path is shadowed. +sys.path.append(str(Path(__file__).resolve().parents[2])) + +import runner # noqa: E402 + +RUN_TS = 1791701190 # 2026-10-11T07:26:30Z +RECORD_SELECT = "has(line.msg) && line.msg == 'sweperf task record' && has(line.v) && line.v == 1.0" +EVALUATIONS = [{ + "name": "goodput_strict", + "variables": [{"name": "goodput", "source": "cel", "source_cfg": { + "inputs": [ + {"name": "records", "reader": "json_lines", + "reader_cfg": {"file": "logs.txt", "prefix": "[boomer] ", "select": RECORD_SELECT}}, + {"name": "one_user", "from_test": "t1", "reader": "json_lines", + "reader_cfg": {"file": "logs.txt", "prefix": "[boomer] ", "select": RECORD_SELECT}}, + ], + "locals": [ + {"name": "ref_wall_ms", "cel": "percentile(one_user.lines.filter(t, !t.failed).map(t, t.wall_ms), 50.0)"}, + {"name": "good", + "cel": "records.lines.filter(t, !t.failed && t.resume_max_ms <= 1500.0 && t.wall_ms <= 1.5 * ref_wall_ms)"}, + ], + "values": [ + {"name": "good_ratio_pct", "cel": "round(100.0 * double(size(good)) / double(size(records.lines)), 2)"}, + {"name": "ref_wall_ms", "cel": "ref_wall_ms"}, + {"name": "one_user_run", "cel": "one_user.uri"}, + ], + }}], + "conditions": [{"name": "good_ratio", "cel": "goodput.good_ratio_pct >= 90"}], +}] + + +def boomer(ts: int, msg: str, **kw) -> str: + t = datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + return "[boomer] " + json.dumps({"time": t, "msg": msg, **kw}) + + +def write_run(logs: Path, stats: Path, run_ts: int, wall_ms: float) -> None: + """What a run of runner.py leaves: one sweperf task record, and a trial_summary row.""" + end = datetime.fromtimestamp(run_ts + 30, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + logs.parent.mkdir(parents=True, exist_ok=True) + logs.write_text("\n".join([ + boomer(run_ts + 30, "sweperf task record", v=1, actor="a1", end=end, cel_ms=6000.0, wall_ms=wall_ms, + resume_max_ms=200.0, failed=False), + boomer(run_ts + 60, "sweperf shutdown started")]) + "\n") + trial = {"timestamp": "", "tag": "abc", "test_name": "t", "metric": "trial_summary", + "measurements": {"allocatable_cores": "100", "run_start": str(run_ts), "run_end": str(run_ts + 60)}} + stats.write_text(json.dumps(trial) + "\n") + + +def args(evaluations: object, dest: str) -> argparse.Namespace: + return argparse.Namespace(test_entry={"name": "t150", "type": "locust", "evaluations": evaluations}, tag="abc", + name="t150", dest=dest) + + +class EvaluationNamesTest(unittest.TestCase): + def test_names(self): + self.assertEqual(runner.evaluation_names(None), "(none)") + self.assertEqual(runner.evaluation_names(EVALUATIONS * 2), "goodput_strict, goodput_strict") + self.assertEqual(runner.evaluation_names([]), "(empty)") + self.assertEqual(runner.evaluation_names("oops"), "(unreadable, see below)") + + +class RunEvaluationsTest(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.dest = Path(tmp.name) / "dest" + self.work = Path(tmp.name) / "work" + self.files = {"logs.txt": self.work / "t150_logs.txt", "stats.jsonl": self.work / "t150.jsonl"} + self.run_dir = self.work / "run" + runner.link_artifacts([(path, name) for name, path in self.files.items()], self.run_dir) + self.identity = runner.run_identity(args(None, str(self.dest)), "2026-10-11", RUN_TS, + f"{self.dest}/runs/t150/run_date=2026-10-11/run_ts={RUN_TS}/run_tag=abc") + + def run_step(self, a: argparse.Namespace) -> tuple[dict, list[str], str]: + """prepare_evaluations at the start of the run, then the run, then run_evaluations.""" + logs = io.StringIO() + evaluator, error = runner.prepare_evaluations(a, self.run_dir, self.identity, logs) + write_run(self.files["logs.txt"], self.files["stats.jsonl"], RUN_TS, wall_ms=8000.0) + out = self.work / "t150_evaluation.json" + runner.run_evaluations(a, evaluator, error, self.identity, out, self.files["stats.jsonl"], "ts", logs) + metrics = [json.loads(line)["metric"] for line in self.files["stats.jsonl"].read_text().splitlines()] + return json.loads(out.read_text()), metrics, logs.getvalue() + + def test_judges_the_run_and_appends_rows(self): + report, metrics, log = self.run_step(args(EVALUATIONS, str(self.dest))) + self.assertEqual(report["run"], {"test_name": "t150", "tag": "abc", "run_date": "2026-10-11", + "run_ts": str(RUN_TS), "uri": self.identity.uri}) + variable = report["evaluations"][0]["variables"][0] + self.assertEqual((report["summary"]["status"], variable["values"]), ("invalid", {})) + # No run of t1 under --dest: the input can't be read, and the reason says why. + self.assertEqual(variable["invalid_reason"], f"input 'one_user': no run of t1 with tag abc under " + f"{self.dest}/runs/ in the 6 h before run_ts {RUN_TS}") + self.assertEqual(metrics, ["trial_summary", "evaluation_goodput_strict", "evaluations"]) + self.assertIn("evaluation goodput_strict: invalid (invalid: good_ratio; " + "variable goodput is invalid: input 'one_user'", log) + + def test_paired_run_from_dest(self): + paired = self.dest / "runs/t1/run_date=2026-10-11" / f"run_ts={RUN_TS - 600}/run_tag=abc" + write_run(paired / "logs.txt", paired / "stats.jsonl", RUN_TS - 600, wall_ms=4000.0) + report, _, log = self.run_step(args(EVALUATIONS, str(self.dest))) + values = report["evaluations"][0]["variables"][0]["values"] + self.assertEqual((values["one_user_run"], values["ref_wall_ms"]), (str(paired), 4000.0)) + # 8000 ms against 1.5 x the 1-user run's 4000 ms. + self.assertEqual((values["good_ratio_pct"], report["summary"]["status"]), (0.0, "fail")) + self.assertIn("evaluation goodput_strict: fail (failed: good_ratio)", log) + + def test_prepare_never_raises(self): + logs = io.StringIO() + with unittest.mock.patch("evaluate.check_config", side_effect=TypeError("unhashable type: 'list'")): + evaluator, error = runner.prepare_evaluations(args([], str(self.dest)), self.run_dir, self.identity, logs) + self.assertIsNone(evaluator) + self.assertEqual(error, "evaluate failed to load or prepare the evaluations: TypeError: unhashable type: 'list'") + self.assertIn("Warning: evaluations will not be judged", logs.getvalue()) + + def test_config_error_is_found_before_the_run_and_recorded(self): + report, metrics, log = self.run_step(args([{"name": "Bad"}], str(self.dest))) + self.assertTrue(report["error"].startswith("config: evaluations: 0.name: String should match pattern")) + self.assertEqual((report["summary"]["status"], report["evaluations"]), ("invalid", [])) + self.assertEqual(metrics, ["trial_summary", "evaluations"]) + self.assertIn("Warning: evaluations will not be judged: config: ", log) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarking/locust/unit_tests/test_runner_test_entry.py b/benchmarking/locust/unit_tests/test_runner_test_entry.py new file mode 100644 index 0000000000..a04864fc14 --- /dev/null +++ b/benchmarking/locust/unit_tests/test_runner_test_entry.py @@ -0,0 +1,36 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for runner.py's --test-entry parsing.""" + +import argparse +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import runner # noqa: E402 + + +class TestEntryTest(unittest.TestCase): + def test_json_object(self): + self.assertEqual(runner.json_object('{"name": "t", "observers": []}'), {"name": "t", "observers": []}) + for bad in ("{oops", "[]"): + with self.subTest(bad=bad), self.assertRaises(argparse.ArgumentTypeError): + runner.json_object(bad) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/guides/benchmarking.md b/docs/guides/benchmarking.md index 35c4caff6e..238876fd6e 100644 --- a/docs/guides/benchmarking.md +++ b/docs/guides/benchmarking.md @@ -251,9 +251,11 @@ a run does not show up as a burst of failures. #### Sweperf Log Lines -The boomer log also carries per-task facts for post-processing such as the goodput script in -`tools/goodput`, which decides from them whether a task was good. The benchmark judges -nothing. Durations are in milliseconds. +The boomer log also carries per-task facts for post-processing, which run +evaluations can read with the `cel` source's `json_lines` reader +(`benchmarking/evaluate/README.md`), where a test entry's config decides +whether a task was good. The benchmark judges nothing. Durations are in +milliseconds. * `Completed sweperf cycle`: `actor`, `cycle`, `cel_ms` (0 when the execute failed), `wall_ms` (resume start to cycle end), `ok` (the execute succeeded, and so did the suspend @@ -273,6 +275,8 @@ nothing. Durations are in milliseconds. the no-op resume got an asynchronous ack, so always for a one-cycle task), `retries` (failed resumes), `failed`, and the first failure's `fail_step` (`resume`, `execute` or `suspend`) and `fail_cycle`. +* `SuspendActorAsync failed` (`SuspendActor failed` in the sync lifecycle mode): `actor` and + `err`. Like the other failure lines, it ties a task that never logged a record to its actor. * `sweperf on_start failed`: also carries `user`, to tie a failed start to its user. * `sweperf shutdown started`: the worker began tearing down. Startups and cycles in flight are canceled: interrupted tasks log no task record, and startup failures logged after it @@ -846,6 +850,14 @@ them are checked into the repository. [analysis/README.md](../../benchmarking/analysis/README.md#automated-runs)). * `server_summary.json`: server-side telemetry harvested from Prometheus, including the per-sample bin-packing timeseries. +* `evaluation.json`: for a test entry with `evaluations`, the run judged + against them: a summary verdict, and per evaluation each condition with its + CEL expression and verdict and every value its sources read. `stats.jsonl` + then also has one `evaluation_` row per evaluation and an + `evaluations` row for the summary. Evaluations never change `status.json` or + the exit code. See [evaluate/README.md](../../benchmarking/evaluate/README.md); + to judge a stored run again, run `python3 -m evaluate --from-run` from + `benchmarking/`. ### Density frontiers