Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions benchmarking/automation/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -165,13 +165,18 @@ gcloud projects add-iam-policy-binding <TEST_PROJECT_ID> \
```

**Runner Job pod** (KSA `benchmark-runner` in namespace `benchmarking` on the
test cluster's project) needs `roles/storage.objectUser` on the destination
bucket so `runner.py` can upload results:
test cluster's project) needs `roles/storage.objectCreator` on the destination
bucket so `runner.py` can upload results, and `roles/storage.objectViewer` so
that a `cel` input with a `test` can find and read that test's run (such as
the suite's 1-user run) under `--dest` (without it, the variable is
`invalid` and says why):

```bash
RUNNER_PRINCIPAL="principal://iam.googleapis.com/projects/<TEST_PROJECT_NUMBER>/locations/global/workloadIdentityPools/<TEST_PROJECT_ID>.svc.id.goog/subject/ns/benchmarking/sa/benchmark-runner"
gcloud storage buckets add-iam-policy-binding gs://<DEST_BUCKET> \
--role=roles/storage.objectCreator --member="${RUNNER_PRINCIPAL}"
gcloud storage buckets add-iam-policy-binding gs://<DEST_BUCKET> \
--role=roles/storage.objectViewer --member="${RUNNER_PRINCIPAL}"
```

## Updating tests
Expand Down
6 changes: 6 additions & 0 deletions benchmarking/automation/orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,9 @@ def validate_and_normalize_tests(tests: list[dict[str, Any]]) -> None:
if ttype == "locust":
# A malformed --metadata fails here rather than mid-sweep.
split_metadata_flag(t)
elif "evaluations" in t:
# Only the locust runner judges evaluations (runner.py --test-entry).
raise ValueError(f"test {name!r}: 'evaluations' is supported only for type locust")
isolate_block(t)


Expand Down Expand Up @@ -516,6 +519,9 @@ def run_test(
if test_type(test) == "locust":
flags, own = split_metadata_flag(test)
flags += ["--metadata", json.dumps({**run_metadata(), **own})]
# The whole entry, so runner features (such as observers or evaluations)
# read what they need without adding a new flag per field.
flags += ["--test-entry", json.dumps(test)]
manifest = render_template(tmpl, subs, flags)
wait_for_no_active_runners()
print(f"Submitting Job {job}", flush=True)
Expand Down
7 changes: 7 additions & 0 deletions benchmarking/automation/test_locust.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,13 @@ def test_missing_required(self):
with self.assertRaises(ValueError):
locust.validate(test)

# Shape only: the runner validates the rest (benchmarking/evaluate).
def test_evaluations_shape(self):
locust.validate(entry(evaluations=[{"name": "e", "anything": "else"}]))
for bad in ({"evaluations": {"name": "e"}}, {"evaluations": [{"name": ""}]}, {"evaluations": ["e"]}):
with self.subTest(bad=bad), self.assertRaisesRegex(ValueError, "locust test 't': evaluations must"):
locust.validate(entry(**bad))


class PreTestTest(unittest.TestCase):
def shaped(self, test):
Expand Down
20 changes: 20 additions & 0 deletions benchmarking/automation/test_orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,26 @@ def test_bad_test_metadata_fails_clearly(self):
with self.assertRaisesRegex(ValueError, "not valid JSON"):
orchestrator.validate_and_normalize_tests([test])

def test_locust_job_gets_the_whole_test_entry(self):
class Rendered(Exception):
pass

fake = mock.Mock(job_tmpl=mock.Mock(return_value="t"), job_subs=mock.Mock(return_value={}))
test = {"name": "n", "type": "locust", "duration": "1m", "flags": ["--x"], "observers": [{"name": "pprof"}]}
with mock.patch.dict(orchestrator.TYPES, {"locust": fake}), \
mock.patch.dict(os.environ, CLUSTER_ENV, clear=True), \
mock.patch.object(orchestrator, "render_template", side_effect=Rendered) as render, \
self.assertRaises(Rendered):
orchestrator.run_test(test, "img", "dest", "abcdef0123")
args = render.call_args.args[2]
self.assertEqual(json.loads(args[args.index("--test-entry") + 1]), test)

def test_evaluations_only_on_locust(self):
test = {"name": "n", "type": "nighthawk-ingress", "targetCluster": "t", "evaluations": [{"name": "e"}]}
with mock.patch.object(orchestrator.TYPES["nighthawk-ingress"], "validate"), \
self.assertRaisesRegex(ValueError, "test 'n': 'evaluations' is supported only for type locust"):
orchestrator.validate_and_normalize_tests([test])

def test_metadata_survives_yaml(self):
md = json.dumps({"cluster_name": "c: d", "q": "\"'"})
subs = {"JOB_NAME": "j", "IMAGE": "i", "TAG": "t", "NAME": "n", "DEST": "d"}
Expand Down
19 changes: 19 additions & 0 deletions benchmarking/automation/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,25 @@
# seconds, or the step falls short of its rate. A resumepaused iteration is a
# resume plus a pause, so size it by the p99 of both together.
#
# Optional `evaluations` judges the run against conditions on its results:
# runner.py writes the verdicts to evaluation.json and adds rows to
# stats.jsonl (evaluation_<name>, and evaluations for the summary); they never
# fail the run. The orchestrator passes the whole entry to runner.py as
# --test-entry. A cel input's `test` names another test of the suite (such
# as its 1-user test), whose latest run under the same --dest with the same
# tag (at most 6h older, on the same cluster) it reads. See
# benchmarking/evaluate/README.md; check a file with
# `cd benchmarking && python3 -m evaluate --check-config --tests-file F`.
#
# evaluations:
# - name: envelope
# variables:
# - name: trial
# source: run_results
# source_cfg: {metric: trial_summary, measurements: [aggregate_failure_ratio]}
# conditions:
# - {name: failure_ratio, cel: "trial.aggregate_failure_ratio <= 0.001"}
#
# An optional `controlPlane:` block shapes the control plane for a capacity
# run; pre_test applies it after the deploy. Its `apiServer:` section, which
# is required, sizes ate-api-server:
Expand Down
17 changes: 17 additions & 0 deletions benchmarking/automation/testtypes/locust.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@
An optional `controlPlane:` block shapes ate-api-server and Postgres for a
capacity run (see controlplane.py); without it pre_test leaves the cluster
as installed.

The runner gets the whole entry (--test-entry) and judges the run against
its optional `evaluations:` (benchmarking/evaluate/README.md). This image
doesn't ship the evaluate package, so validate checks their shape only; the
runner checks the rest before its run, and `python -m evaluate
--check-config` checks a tests file offline.
"""

import os
Expand All @@ -38,11 +44,22 @@ def control_plane_config(test: dict[str, Any]) -> dict[str, Any] | None:
return controlplane.config(test["controlPlane"])


def validate_evaluations_shape(test: dict[str, Any]) -> None:
"""Raises ValueError unless evaluations is a list of mappings with a name."""
name = test.get("name")
evaluations = test.get("evaluations", [])
if not isinstance(evaluations, list) or not all(
isinstance(e, dict) and isinstance(e.get("name"), str) and e["name"] for e in evaluations
):
raise ValueError(f"locust test {name!r}: evaluations must be a list of mappings, each with a name")


def validate(test: dict[str, Any]) -> None:
name = test.get("name")
for field in ("file", "duration", "users"):
if field not in test:
raise ValueError(f"locust test {name!r} missing {field!r}")
validate_evaluations_shape(test)
if "controlPlane" not in test:
return
try:
Expand Down
Loading
Loading