From df0be56fe01363dd2fa90e34f86c3fa05eabf275 Mon Sep 17 00:00:00 2001 From: Nishanth Kotla Date: Wed, 4 Nov 2026 06:15:14 +0000 Subject: [PATCH] benchmarking/sweperf: optionally retry ResumeActor when no worker has room Add --sweperf-resume-retry-budget (default 0, off). With a budget, a cycle retries ResumeActor on ResourceExhausted or Unavailable with 50ms-to-4s backoff until the budget runs out. Retried refusals are recorded on ResumeActorRefused and only the final attempt on ResumeActor. In the task record, resume_max_ms includes the wait when a resume was retried, and capacity_retries counts the refusals. Signed-off-by: Nishanth Kotla --- benchmarking/locust/common/boomer_config.py | 1 + benchmarking/locust/common/sweperf_config.py | 12 + .../locust/unit_tests/test_boomer_config.py | 3 + docs/guides/benchmarking.md | 38 ++- .../benchmarking/boomer/sweperf/sweperf.go | 214 +++++++++++++--- .../boomer/sweperf/sweperf_test.go | 233 +++++++++++++++++- 6 files changed, 448 insertions(+), 53 deletions(-) diff --git a/benchmarking/locust/common/boomer_config.py b/benchmarking/locust/common/boomer_config.py index cf38419472..0ceca948d1 100644 --- a/benchmarking/locust/common/boomer_config.py +++ b/benchmarking/locust/common/boomer_config.py @@ -75,6 +75,7 @@ class that consumes it. "--sweperf-total-steps": int, "--sweperf-num-cycles": int, "--sweperf-poll-interval-ms": int, + "--sweperf-resume-retry-budget": float, "--agentsession-script": str, "--agentsession-script-file": str, "--agentsession-think-scale": float, diff --git a/benchmarking/locust/common/sweperf_config.py b/benchmarking/locust/common/sweperf_config.py index e4810ca870..302d71c7f9 100644 --- a/benchmarking/locust/common/sweperf_config.py +++ b/benchmarking/locust/common/sweperf_config.py @@ -52,3 +52,15 @@ def add_sweperf_arguments(parser: LocustArgumentParser) -> None: help="Interval in ms between /status polls while a cycle's job runs (default: 100)", include_in_web_ui=True, ) + parser.add_argument( + "--sweperf-resume-retry-budget", + type=float, + default=0.0, + env_var="LOCUST_SWEPERF_RESUME_RETRY_BUDGET", + help=( + "Seconds a cycle retries ResumeActor on capacity errors " + "(ResourceExhausted or Unavailable) before giving up " + "(default: 0, single attempt)" + ), + include_in_web_ui=True, + ) diff --git a/benchmarking/locust/unit_tests/test_boomer_config.py b/benchmarking/locust/unit_tests/test_boomer_config.py index c6a8a4630d..6f4362225c 100644 --- a/benchmarking/locust/unit_tests/test_boomer_config.py +++ b/benchmarking/locust/unit_tests/test_boomer_config.py @@ -55,10 +55,13 @@ def test_build_config_json_phase_in_and_retry_budget(self): "300", "--agentsession-resume-retry-budget", "0", + "--sweperf-resume-retry-budget", + "60", ]) parsed = json.loads(raw) self.assertEqual(parsed["agentsession_phase_in"], 300.0) self.assertEqual(parsed["agentsession_resume_retry_budget"], 0.0) + self.assertEqual(parsed["sweperf_resume_retry_budget"], 60.0) def test_build_config_json_turnover(self): raw = boomer_config.build_config_json([ diff --git a/docs/guides/benchmarking.md b/docs/guides/benchmarking.md index 178ee6c5c4..ef57f8afb3 100644 --- a/docs/guides/benchmarking.md +++ b/docs/guides/benchmarking.md @@ -210,11 +210,26 @@ Then start the run with `--sweperf-template sweperf-django-11099` and and `LOCUST_SWEPERF_TOTAL_STEPS`. The client takes the step count from that option, not from the template, so it must match the catalog. +* `--sweperf-resume-retry-budget`: seconds a cycle keeps retrying a `ResumeActor` the fleet + had no room for (`ResourceExhausted` or `Unavailable`), with 50ms-to-4s backoff, before + giving the cycle up. Default `0` (off): every refusal fails the task, so overload shows up + as task failures. Keep it at `0` for goodput and host-scale sweeps. Set it (about the size + of the wake SLO) only to count completed tasks on a capacity-limited run; refusals then + become wake latency, so read `ResumeWithRetry` as the wake time. + #### Sweperf Reported Metrics All rows are in milliseconds. CEL (command execution latency) is the time the trace commands ran inside the sandbox, as reported by `replay.py`. +* `ResumeWithRetry`: Full resume wall time from the first `ResumeActor` attempt until + `ResumeActor` finishes, including any capacity-retry backoff wait (only the final attempt + is recorded on `ResumeActor`; retried refusals are on `ResumeActorRefused`). +* `ResumeActorRefused`: retried `ResumeActor` attempts the fleet had no room for (only with a + retry budget); its requests/s is the refused attempt rate. These attempts are not recorded + on `ResumeActor` or `ResumeActor_rtt`. Not on the dashboard yet. +* `ResumeRetryGiveUp`: Resumes that still failed with `ResourceExhausted` or `Unavailable` + after `--sweperf-resume-retry-budget` expired. * `ResumeToFirstExec`: Resume RPC start until the sandbox accepts the cycle's `/execute`. * `CycleCEL`: CEL for one cycle. * `TaskCEL`: CEL summed over one task. @@ -230,7 +245,8 @@ ran inside the sandbox, as reported by `replay.py`. The liveness check on a new actor already has it running, so the first cycle of every task runs on a freshly started actor and its resume is a no-op. Its successful `ResumeActor`, -`ResumeActor_rtt` and `ResumeToFirstExec` samples are not recorded; failures still are. +`ResumeActor_rtt`, `ResumeWithRetry` and `ResumeToFirstExec` samples are not recorded; +failures still are. The last cycle of a task is not suspended: the actor is deleted straight from running. A task of `n` cycles therefore records `n - 1` `SuspendActor` samples, and `TaskWallClock` @@ -257,11 +273,13 @@ nothing. Durations are in milliseconds. * `Completed sweperf cycle`: `actor`, `cycle`, `cel_ms` (0 when the execute failed), `wall_ms` (resume start to cycle end), `ok` (the execute succeeded, and so did the suspend - if there was one), `resume_ms` (as on the `ResumeActor` row; omitted for a no-op resume) - and `ttfe_ms` (resume start to the asynchronous ack; omitted without one, and for a no-op - resume even when its execute got one). -* `ResumeActor failed in step`: also carries `wall_ms`. The cycle is retried on the next - iteration, and the failed attempt's time counts toward the task's `wall_ms`. + if there was one), `resume_ms` (as on the `ResumeActor` row), `resume_with_retry_ms` (as + on the `ResumeWithRetry` row) and `resume_retries` (refused attempts retried), all three + omitted for a no-op resume, and `ttfe_ms` (resume start to the asynchronous ack; omitted + without one, and for a no-op resume even when its execute got one). +* `ResumeActor failed in step`: also carries `wall_ms`, `resume_retries` and `gave_up` (the + retry budget ran out). The cycle is retried on the next iteration, and the failed + attempt's time counts toward the task's `wall_ms`. * `sweperf task record`, one per finished task: `v` (1), `user` (boomer goroutine ID, unique per boomer process), `actor`, `template`, `cycles` and `steps` (the cycle and trace step counts the task ran, resolved when it started, so a mid-run change to @@ -269,9 +287,11 @@ nothing. Durations are in milliseconds. capped at `steps`), `task_start` (when the session started) and `end`, `startup_ms` (`CreateActor` until the sandbox answers `/status`), `cel_ms` and `wall_ms` (as in `TaskCEL` and `TaskWallClock`), `think_ms` (think time between the task's cycles), `resume_max_ms` - (slowest non-no-op resume), `ttfe_max_ms` (slowest `ttfe_ms`; omitted when no cycle after + (slowest non-no-op resume: server time, or client wall time including capacity-retry waits + when the resume was retried), `ttfe_max_ms` (slowest `ttfe_ms`; omitted when no cycle after the no-op resume got an asynchronous ack, so always for a one-cycle task), `retries` - (failed resumes), `failed`, and the first failure's `fail_step` (`resume`, `execute` or + (failed resumes), `capacity_retries` (ResumeActor attempts refused for capacity and retried + within the budget), `failed`, and the first failure's `fail_step` (`resume`, `execute` or `suspend`) and `fail_cycle`. * `sweperf on_start failed`: also carries `user`, to tie a failed start to its user. * `sweperf shutdown started`: the worker began tearing down. Startups and cycles in flight @@ -407,7 +427,7 @@ deployment fails loudly instead of showing up as OOM-flaky steps. the step up until its next think gap (default 60; `0` gives the wake one attempt). Each attempt is its own `ResumeActor` or `WakeFirstTouch` sample; the whole wait lands in - `TimeToWake`. + `TimeToWake` (for `sweperf`, see `--sweperf-resume-retry-budget`). * `--agentsession-awake` — turnover mode when above 0 (default 0, off). Each session registers by creating its actor, running step 1, and parking it. After registration is idle for 1 minute, the worker wakes this many sessions diff --git a/internal/benchmarking/boomer/sweperf/sweperf.go b/internal/benchmarking/boomer/sweperf/sweperf.go index f503c34c08..bf0424bbb4 100644 --- a/internal/benchmarking/boomer/sweperf/sweperf.go +++ b/internal/benchmarking/boomer/sweperf/sweperf.go @@ -29,6 +29,7 @@ import ( "fmt" "io" "log/slog" + mathrand "math/rand/v2" "net/http" "strings" "sync" @@ -68,15 +69,26 @@ const ( // defaultSweperfPollInterval trades CycleCEL-to-ack accuracy against // /status load on the router; sympy cycles take roughly 1.5-15s. defaultSweperfPollInterval = 100 * time.Millisecond + + // Same initial backoff as agentsession's wake retry and glutton's spawn + // retry; the cap is higher because sweperf users wait for room much longer. + resumeRetryInitialBackoff = 50 * time.Millisecond + resumeRetryMaxBackoff = 4 * time.Second + resumeRetryJitterFraction = 0.1 ) // Derived rows: each spans more than one call, hence the actor method. const ( - resumeToFirstExecMetric = "ResumeToFirstExec" - cycleCELMetric = "CycleCEL" - taskCELMetric = "TaskCEL" - taskWallClockMetric = "TaskWallClock" - derivedMetricMethod = "actor" + resumeWithRetryMetric = "ResumeWithRetry" + resumeRetryGiveUpMetric = "ResumeRetryGiveUp" + // resumeActorRefusedMetric is the grpc row for ResumeActor attempts the + // fleet had no room for that were retried within the budget. + resumeActorRefusedMetric = "ResumeActorRefused" + resumeToFirstExecMetric = "ResumeToFirstExec" + cycleCELMetric = "CycleCEL" + taskCELMetric = "TaskCEL" + taskWallClockMetric = "TaskWallClock" + derivedMetricMethod = "actor" ) // init registers the sweperf user class so the boomer worker can select it by @@ -106,6 +118,11 @@ type knobs struct { TotalSteps int `json:"sweperf_total_steps"` NumCycles int `json:"sweperf_num_cycles"` PollIntervalMs int `json:"sweperf_poll_interval_ms"` + // ResumeRetryBudget bounds how long a cycle keeps retrying a ResumeActor + // the fleet had no room for (ResourceExhausted or Unavailable) before giving + // up on the cycle, with backoff from 50ms to 4s. 0 (the default) gives the + // resume one attempt. + ResumeRetryBudget dynconfig.Seconds `json:"sweperf_resume_retry_budget"` } var knobsCodec = dynconfig.Typed[knobs]{ @@ -122,6 +139,9 @@ var knobsCodec = dynconfig.Typed[knobs]{ if k.PollIntervalMs < 0 { return fmt.Errorf("sweperf_poll_interval_ms cannot be negative: %d", k.PollIntervalMs) } + if k.ResumeRetryBudget < 0 { + return fmt.Errorf("sweperf_resume_retry_budget cannot be negative: %v", k.ResumeRetryBudget) + } if err := k.Lifecycle.Validate(); err != nil { return err } @@ -544,10 +564,15 @@ type sweperfUser struct { startupDur time.Duration // CreateActor until the sandbox answered /status loopThink time.Duration // think time between this task's cycles loopRetries int // failed resumes, each of which retries its cycle - resumeMax time.Duration // slowest non-no-op ResumeActor, as recorded on its row - ttfeMax time.Duration // slowest resume-to-ack; zero when no cycle after the no-op resume got an async ack - failStep string // first failed step: resume, execute or suspend - failCycle int // cycle of failStep + // capacityRetries counts ResumeActor attempts refused for capacity and + // retried within the budget. + capacityRetries int + // resumeMax is the slowest non-no-op resume: server time, or client wall + // time including waits when the resume was retried. + resumeMax time.Duration + ttfeMax time.Duration // slowest resume-to-ack; zero when no cycle after the no-op resume got an async ack + failStep string // first failed step: resume, execute or suspend + failCycle int // cycle of failStep } // ref is the control-plane reference to this user's actor. @@ -598,21 +623,98 @@ func (u *sweperfUser) create(ctx context.Context) error { }) } -// resume wakes the actor for the next cycle and reports the latency recorded -// on its ResumeActor row and whether it worked. A failure is the caller's cue -// to abandon the cycle rather than talk to a sandbox that is not running. -func (u *sweperfUser) resume(ctx context.Context) (time.Duration, bool) { - latency, err := u.tracedCallRecord(ctx, "ResumeActor", !u.awake, func(callCtx context.Context, tr *metadata.MD) error { - _, err := u.cfg.APIStub.ResumeActor(callCtx, &ateapi.ResumeActorRequest{ - Actor: u.ref(), - }, grpc.Trailer(tr)) - return err - }) - if err != nil { - slog.Error("ResumeActor failed", slog.String("actor", u.actorName), slog.String("err", err.Error())) - return latency, false +// resumeOutcome holds the timing and retry counts of one cycle's resume. +type resumeOutcome struct { + latency time.Duration // latency recorded on the last ResumeActor attempt + total time.Duration // wall time from first attempt to completion, including retry wait + retries int // number of retry attempts after the first (attempts - 1) + ok bool // whether ResumeActor succeeded + gaveUp bool // true when a capacity error ran out of the retry budget +} + +// isCapacityError reports whether err indicates the fleet or control plane had +// no room for the actor right now (ResourceExhausted or Unavailable). It is the +// gRPC half of agentsession's isCapacityError (sweperf's resume is gRPC only). +func isCapacityError(err error) bool { + switch status.Code(err) { + case codes.ResourceExhausted, codes.Unavailable: + return true + } + return false +} + +// resume runs ResumeActor to wake the actor for the next cycle and, while it +// fails with a capacity error (ResourceExhausted or Unavailable) and the next +// attempt starts before sweperf_resume_retry_budget ends, backs off and +// retries. Retried capacity refusals are recorded on ResumeActorRefused (grpc); +// only the final attempt is recorded on ResumeActor and ResumeActor_rtt. Full +// wall time is recorded on ResumeWithRetry, and capacity errors that run out of +// a positive budget are recorded on ResumeRetryGiveUp. +func (u *sweperfUser) resume(ctx context.Context) resumeOutcome { + start := time.Now() + budget := dynconfig.Get[knobs](u.cfg.Dyn).ResumeRetryBudget.Duration() + deadline := start.Add(budget) + var ( + latency time.Duration + err error + retries int + gaveUp bool + ) + delay := resumeRetryInitialBackoff + for { + var ( + clientLatency time.Duration + source string + ) + latency, clientLatency, source, err = u.timedCall(ctx, "ResumeActor", func(callCtx context.Context, tr *metadata.MD) error { + _, err := u.cfg.APIStub.ResumeActor(callCtx, &ateapi.ResumeActorRequest{ + Actor: u.ref(), + }, grpc.Trailer(tr)) + return err + }) + if err != nil && canceled(ctx) { + return resumeOutcome{latency: latency, total: time.Since(start), retries: retries} + } + if err != nil && isCapacityError(err) && budget > 0 { + wait := delay + time.Duration(mathrand.Float64()*resumeRetryJitterFraction*float64(delay)) + if time.Now().Add(wait).Before(deadline) { + bmetrics.RecordFailure("grpc", resumeActorRefusedMetric, u.userClass, latency, err.Error()) + select { + case <-time.After(wait): + case <-ctx.Done(): + return resumeOutcome{latency: latency, total: time.Since(start), retries: retries} + } + retries++ + delay = nextResumeBackoff(delay) + continue + } + gaveUp = true + } + u.recordRPC("ResumeActor", latency, clientLatency, source, err, !u.awake) + break } - return latency, true + + total := time.Since(start) + if err == nil { + if !u.awake { + bmetrics.RecordSuccess(derivedMetricMethod, resumeWithRetryMetric, u.userClass, total, 0) + } + return resumeOutcome{latency: latency, total: total, retries: retries, ok: true} + } + if canceled(ctx) { + return resumeOutcome{latency: latency, total: total, retries: retries} + } + bmetrics.RecordFailure(derivedMetricMethod, resumeWithRetryMetric, u.userClass, total, err.Error()) + if gaveUp { + bmetrics.RecordFailure(derivedMetricMethod, resumeRetryGiveUpMetric, u.userClass, total, err.Error()) + } + slog.Error("ResumeActor failed", + slog.String("actor", u.actorName), + slog.Int("retries", retries), + slog.Bool("gave_up", gaveUp), + slog.Duration("waited", total), + slog.String("err", err.Error())) + return resumeOutcome{latency: latency, total: total, retries: retries, gaveUp: gaveUp} } // suspend snapshots the actor and puts it to sleep, ending a cycle. It reports @@ -674,33 +776,58 @@ func (u *sweperfUser) tracedCall(ctx context.Context, name string, do func(conte // tracedCallRecord is tracedCall with optional success rows; failures are // always recorded. It also returns the latency it records. func (u *sweperfUser) tracedCallRecord(ctx context.Context, name string, recordSuccess bool, do func(context.Context, *metadata.MD) error) (time.Duration, error) { + latency, clientLatency, source, err := u.timedCall(ctx, name, do) + if err != nil && canceled(ctx) { + return latency, err + } + u.recordRPC(name, latency, clientLatency, source, err, recordSuccess) + return latency, err +} + +// timedCall runs one control-plane RPC under a span named name and returns +// its latency (server-measured when the trailer has it, else client), the +// client latency, and whether latency is server- or client-measured. It +// records no stats rows. A call cut short by ctx returns the client latency. +func (u *sweperfUser) timedCall(ctx context.Context, name string, do func(context.Context, *metadata.MD) error) (latency, clientLatency time.Duration, source string, err error) { ctx, span := u.cfg.Tracer.Start(ctx, name) defer span.End() start := time.Now() var tr metadata.MD - err := do(ctx, &tr) - clientLatency := time.Since(start) + err = do(ctx, &tr) + clientLatency = time.Since(start) if err != nil && canceled(ctx) { - return clientLatency, err + return clientLatency, clientLatency, boomerutil.SourceClient, err } - latency, source := boomerutil.ElapsedFromMD(tr, ateinterceptors.ServerElapsedTrailer, clientLatency) + latency, source = boomerutil.ElapsedFromMD(tr, ateinterceptors.ServerElapsedTrailer, clientLatency) if source == boomerutil.SourceServer { span.SetAttributes(attribute.Float64("server.elapsed_ms", boomerutil.MsFloat(latency))) - if err != nil || recordSuccess { - u.recordClientRTT(name, clientLatency, err) - } } boomerutil.LogSampledTrace(span, name, latency, source, err) + return latency, clientLatency, source, err +} + +// recordRPC records one control-plane RPC as a locust stats row of the same +// name, plus a _rtt row with the client latency when latency came from +// the server. Failures are always recorded; successes only if recordSuccess. +func (u *sweperfUser) recordRPC(name string, latency, clientLatency time.Duration, source string, err error, recordSuccess bool) { + if source == boomerutil.SourceServer && (err != nil || recordSuccess) { + u.recordClientRTT(name, clientLatency, err) + } if err != nil { bmetrics.RecordFailure("grpc", name, u.userClass, latency, err.Error()) - return latency, err + return } if recordSuccess { bmetrics.RecordSuccess("grpc", name, u.userClass, latency, 0) } - return latency, nil +} + +// nextResumeBackoff doubles a capacity-retry backoff, up to +// resumeRetryMaxBackoff. +func nextResumeBackoff(d time.Duration) time.Duration { + return min(d*2, resumeRetryMaxBackoff) } // recordClientRTT publishes the client wall clock for an RPC whose own row carries server time. @@ -802,26 +929,36 @@ func (u *sweperfUser) step(ctx context.Context) { // Summed per cycle so the between-cycle think time stays out of the total. defer func() { u.loopWall += time.Since(resumeStart) }() // A cycle cut short by shutdown returns without recording or advancing. - resumeLatency, resumed := u.resume(ctx) - if !resumed { + res := u.resume(ctx) + if !res.ok { if canceled(ctx) { return } u.loopFailed = true u.loopRetries++ + u.capacityRetries += res.retries u.noteFailure("resume", cycleNum) bmetrics.RecordFailure(derivedMetricMethod, resumeToFirstExecMetric, u.userClass, time.Since(resumeStart), "ResumeActor failed") slog.Error("ResumeActor failed in step", slog.String("actor", u.actorName), slog.Int("cycle", cycleNum), + slog.Int("resume_retries", res.retries), + slog.Bool("gave_up", res.gaveUp), slog.Float64("wall_ms", boomerutil.MsFloat(time.Since(resumeStart))), ) return } + resumeLatency := res.latency u.awake = false if !noopResume { - u.resumeMax = max(u.resumeMax, resumeLatency) + d := resumeLatency + if res.retries > 0 { + // Include the capacity-retry waits. + d = res.total + } + u.resumeMax = max(u.resumeMax, d) + u.capacityRetries += res.retries } // 2. Execute step range chunk inside sandbox @@ -874,7 +1011,11 @@ func (u *sweperfUser) step(ctx context.Context) { } // A no-op resume and a reply without an async ack have nothing to report. if !noopResume { - attrs = append(attrs, slog.Float64("resume_ms", boomerutil.MsFloat(resumeLatency))) + attrs = append(attrs, + slog.Float64("resume_ms", boomerutil.MsFloat(resumeLatency)), + slog.Float64("resume_with_retry_ms", boomerutil.MsFloat(res.total)), + slog.Int("resume_retries", res.retries), + ) } if ttfe > 0 { attrs = append(attrs, slog.Float64("ttfe_ms", boomerutil.MsFloat(ttfe))) @@ -953,6 +1094,7 @@ func (u *sweperfUser) logTaskRecord() { slog.Float64("think_ms", boomerutil.MsFloat(u.loopThink)), slog.Float64("resume_max_ms", boomerutil.MsFloat(u.resumeMax)), slog.Int("retries", u.loopRetries), + slog.Int("capacity_retries", u.capacityRetries), slog.Bool("failed", u.loopFailed), slog.String("fail_step", u.failStep), slog.Int("fail_cycle", u.failCycle), diff --git a/internal/benchmarking/boomer/sweperf/sweperf_test.go b/internal/benchmarking/boomer/sweperf/sweperf_test.go index dcf72695ad..68e197eb48 100644 --- a/internal/benchmarking/boomer/sweperf/sweperf_test.go +++ b/internal/benchmarking/boomer/sweperf/sweperf_test.go @@ -102,12 +102,15 @@ type fakeControlClient struct { createSpaceErr error createActorErr error resumeErr error + resumeErrs []error suspendErr error deleteErr error suspendReqs []*ateapi.SuspendActorRequest // Make ResumeActor take time and report a smaller server-side elapsed. resumeDelay time.Duration resumeElapsedUs string + // onResume, if set, runs on each ResumeActor call. + onResume func() // beforeDelete, if set, runs at the start of each DeleteActor, outside mu. beforeDelete func() } @@ -136,10 +139,20 @@ func (f *fakeControlClient) ResumeActor(ctx context.Context, in *ateapi.ResumeAc f.mu.Lock() defer f.mu.Unlock() f.calls = append(f.calls, "ResumeActor") + if f.onResume != nil { + f.onResume() + } time.Sleep(f.resumeDelay) if f.resumeElapsedUs != "" { setTrailer(opts, metadata.Pairs(ateinterceptors.ServerElapsedTrailer, f.resumeElapsedUs)) } + if len(f.resumeErrs) > 0 { + err := f.resumeErrs[0] + f.resumeErrs = f.resumeErrs[1:] + if err != nil { + return nil, err + } + } if f.resumeErr != nil { return nil, f.resumeErr } @@ -520,7 +533,7 @@ func TestControlClientErrors(t *testing.T) { fakeCtrl.resumeErr = errors.New("resume RPC failed") u := &sweperfUser{cfg: cfg, actorName: "act"} - if _, ok := u.resume(context.Background()); ok { + if u.resume(context.Background()).ok { t.Errorf("resume = true on RPC failure, want false") } }) @@ -801,7 +814,7 @@ func TestNoopFirstResumeNotRecorded(t *testing.T) { t.Errorf("recordedCalls after startUser = %v, want %v", got, want) } u.chunks = []chunk{{0, 5}, {5, 10}} - resumeRows := []string{"ResumeActor", "ResumeActor_rtt", "ResumeToFirstExec"} + resumeRows := []string{"ResumeActor", "ResumeActor_rtt", "ResumeWithRetry", "ResumeToFirstExec"} before := readStats(t) u.step(context.Background()) @@ -836,7 +849,7 @@ func TestNoopFirstResumeFailureStillRecorded(t *testing.T) { before := readStats(t) u.step(context.Background()) - for _, name := range []string{"ResumeActor", "ResumeToFirstExec"} { + for _, name := range []string{"ResumeActor", "ResumeWithRetry", "ResumeToFirstExec"} { if got := recordedSince(t, before, name, "failure"); got.count != 1 { t.Errorf("%s recorded %d failures, want 1", name, got.count) } @@ -889,7 +902,7 @@ func TestTracedCallRecordsBothLatencies(t *testing.T) { u := &sweperfUser{cfg: cfg, actorName: "act", userClass: sweperfUserClass} before := readStats(t) - if _, ok := u.resume(context.Background()); !ok { + if !u.resume(context.Background()).ok { t.Fatalf("resume failed") } @@ -912,7 +925,7 @@ func TestTracedCallSkipsRTTRowWithoutTrailer(t *testing.T) { u := &sweperfUser{cfg: cfg, actorName: "act", userClass: sweperfUserClass} before := readStats(t) - if _, ok := u.resume(context.Background()); !ok { + if !u.resume(context.Background()).ok { t.Fatalf("resume failed") } @@ -1329,10 +1342,10 @@ func TestTracedCallSkipsFailureWhenCanceled(t *testing.T) { cancel() before := readStats(t) - if _, ok := u.resume(ctx); ok { + if u.resume(ctx).ok { t.Fatalf("resume = true on RPC failure, want false") } - for _, name := range []string{"ResumeActor", "ResumeActor_rtt"} { + for _, name := range []string{"ResumeActor", "ResumeActor_rtt", "ResumeWithRetry"} { if got := recordedSince(t, before, name, "failure"); got.count != 0 { t.Errorf("%s recorded %d failures for a canceled call, want 0", name, got.count) } @@ -1371,7 +1384,7 @@ func TestShutdownMidCycleRecordsNoFailures(t *testing.T) { r.shutdown(ctx) for _, name := range []string{ - "ResumeActor", "ResumeActor_rtt", "ResumeToFirstExec", "Workload_Cycle_1", + "ResumeActor", "ResumeActor_rtt", "ResumeActorRefused", "ResumeWithRetry", "ResumeRetryGiveUp", "ResumeToFirstExec", "Workload_Cycle_1", "CycleCEL", "TaskCEL", "TaskWallClock", "SuspendActor", "DeleteActor", } { if got := recordedSince(t, before, name, "failure"); got.count != 0 { @@ -1759,3 +1772,207 @@ func TestShutdownCutsOffInFlightCycle(t *testing.T) { }) } } + +func TestResumeRetryWhileFleetIsFull(t *testing.T) { + t.Run("retries capacity errors within budget and records refusals apart from the final attempt", func(t *testing.T) { + logs := captureLogs(t) + cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) + fakeCtrl.resumeErrs = []error{ + status.Error(codes.ResourceExhausted, "no worker has room for the actor"), + status.Error(codes.Unavailable, "worker transiently unavailable"), + } + cfg.Dyn = dynconfig.Static(knobs{ + ResumeRetryBudget: dynconfig.Seconds(5 * time.Second), + }) + u := &sweperfUser{ + cfg: cfg, + actorName: "act-retry", + userClass: sweperfUserClass, + chunks: []chunk{{0, 5}}, + } + + before := readStats(t) + u.step(context.Background()) + + if u.cycleIndex != 1 || u.loopFailed { + t.Fatalf("after retried resume: cycleIndex=%d loopFailed=%v, want 1/false", u.cycleIndex, u.loopFailed) + } + if got := recordedSince(t, before, "ResumeActorRefused", "failure"); got.count != 2 { + t.Errorf("ResumeActorRefused failures = %d, want 2", got.count) + } + if got := recordedSince(t, before, "ResumeActor", "failure"); got.count != 0 { + t.Errorf("ResumeActor failures = %d, want 0 (retried refusals go to ResumeActorRefused)", got.count) + } + if got := recordedSince(t, before, "ResumeActor", "success"); got.count != 1 { + t.Errorf("ResumeActor successes = %d, want 1", got.count) + } + if got := recordedSince(t, before, "ResumeWithRetry", "success"); got.count != 1 { + t.Errorf("ResumeWithRetry successes = %d, want 1", got.count) + } + if got := recordedSince(t, before, "ResumeRetryGiveUp", "failure"); got.count != 0 { + t.Errorf("ResumeRetryGiveUp failures = %d, want 0", got.count) + } + cyc := onlyLine(t, logs(), completedCycleMsg) + if cyc["resume_retries"] != 2.0 { + t.Errorf("cycle log resume_retries = %v, want 2", cyc["resume_retries"]) + } + u.recordTaskMetrics() + rec := onlyLine(t, logs(), taskRecordMsg) + if rec["capacity_retries"] != 2.0 { + t.Errorf("task record capacity_retries = %v, want 2", rec["capacity_retries"]) + } + // The two waits alone are at least 50ms + 100ms. + if got, _ := rec["resume_max_ms"].(float64); got < 150 { + t.Errorf("task record resume_max_ms = %v, want >= 150 (includes the retry waits)", rec["resume_max_ms"]) + } + }) + + t.Run("gives up after retry budget and records ResumeRetryGiveUp", func(t *testing.T) { + logs := captureLogs(t) + cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) + fakeCtrl.resumeErr = status.Error(codes.ResourceExhausted, "no worker has room for the actor") + const budget = 180 * time.Millisecond + cfg.Dyn = dynconfig.Static(knobs{ + ResumeRetryBudget: dynconfig.Seconds(budget), + }) + u := &sweperfUser{ + cfg: cfg, + actorName: "act-giveup", + userClass: sweperfUserClass, + chunks: []chunk{{0, 5}}, + } + + before := readStats(t) + u.step(context.Background()) + + if u.cycleIndex != 0 || !u.loopFailed { + t.Fatalf("after give-up: cycleIndex=%d loopFailed=%v, want 0/true", u.cycleIndex, u.loopFailed) + } + // Timing-dependent: the 50ms and 100ms waits fit in the budget unless + // the calls themselves are slow. + if got := recordedSince(t, before, "ResumeActorRefused", "failure"); got.count < 1 || got.count > 2 { + t.Errorf("ResumeActorRefused failures = %d, want 1 or 2 within %v budget", got.count, budget) + } + if got := recordedSince(t, before, "ResumeActor", "failure"); got.count != 1 { + t.Errorf("ResumeActor failures = %d, want 1 (the attempt that gave up)", got.count) + } + if got := recordedSince(t, before, "ResumeWithRetry", "failure"); got.count != 1 { + t.Errorf("ResumeWithRetry failures = %d, want 1", got.count) + } + if got := recordedSince(t, before, "ResumeRetryGiveUp", "failure"); got.count != 1 { + t.Errorf("ResumeRetryGiveUp failures = %d, want 1", got.count) + } + failLine := onlyLine(t, logs(), "ResumeActor failed in step") + if failLine["gave_up"] != true { + t.Errorf("failLine gave_up = %v, want true", failLine["gave_up"]) + } + if u.capacityRetries < 1 || u.capacityRetries > 2 { + t.Errorf("capacityRetries = %d after give-up, want 1 or 2", u.capacityRetries) + } + }) + + t.Run("context canceled during capacity error records no failures", func(t *testing.T) { + cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) + ctx, cancel := context.WithCancel(context.Background()) + fakeCtrl.onResume = cancel + fakeCtrl.resumeErr = status.Error(codes.ResourceExhausted, "no worker has room for the actor") + cfg.Dyn = dynconfig.Static(knobs{ + ResumeRetryBudget: dynconfig.Seconds(5 * time.Second), + }) + u := &sweperfUser{ + cfg: cfg, + actorName: "act-canceled", + userClass: sweperfUserClass, + chunks: []chunk{{0, 5}}, + } + + before := readStats(t) + u.step(ctx) + + if u.loopFailed { + t.Errorf("loopFailed = true on canceled resume, want false") + } + for _, name := range []string{"ResumeActor", "ResumeActorRefused", "ResumeWithRetry", "ResumeRetryGiveUp"} { + if got := recordedSince(t, before, name, "failure"); got.count != 0 { + t.Errorf("%s failures = %d on canceled resume, want 0", name, got.count) + } + } + }) + + t.Run("default zero budget does not retry capacity errors", func(t *testing.T) { + cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) + fakeCtrl.resumeErr = status.Error(codes.ResourceExhausted, "no worker has room for the actor") + u := &sweperfUser{ + cfg: cfg, + actorName: "act-nobudget", + userClass: sweperfUserClass, + chunks: []chunk{{0, 5}}, + } + + before := readStats(t) + u.step(context.Background()) + + if got := recordedSince(t, before, "ResumeActor", "failure"); got.count != 1 { + t.Errorf("ResumeActor failures = %d with 0 budget, want 1", got.count) + } + if got := recordedSince(t, before, "ResumeActorRefused", "failure"); got.count != 0 { + t.Errorf("ResumeActorRefused failures = %d with 0 budget, want 0", got.count) + } + if got := recordedSince(t, before, "ResumeRetryGiveUp", "failure"); got.count != 0 { + t.Errorf("ResumeRetryGiveUp failures = %d with 0 budget, want 0", got.count) + } + if u.capacityRetries != 0 { + t.Errorf("capacityRetries = %d with 0 budget, want 0", u.capacityRetries) + } + }) + + t.Run("non-capacity error returns immediately without retry", func(t *testing.T) { + cfg, _, fakeCtrl := newTestConfig(t, asyncJobHandler()) + fakeCtrl.resumeErrs = []error{ + status.Error(codes.ResourceExhausted, "no worker has room"), + status.Error(codes.Aborted, "actor crashed"), + nil, + } + cfg.Dyn = dynconfig.Static(knobs{ + ResumeRetryBudget: dynconfig.Seconds(5 * time.Second), + }) + u := &sweperfUser{ + cfg: cfg, + actorName: "act-aborted", + userClass: sweperfUserClass, + chunks: []chunk{{0, 5}}, + } + + before := readStats(t) + u.step(context.Background()) + + if got := recordedSince(t, before, "ResumeActorRefused", "failure"); got.count != 1 { + t.Errorf("ResumeActorRefused failures = %d, want 1 (the retried capacity error)", got.count) + } + if got := recordedSince(t, before, "ResumeActor", "failure"); got.count != 1 { + t.Errorf("ResumeActor failures = %d, want 1 (the Aborted attempt)", got.count) + } + if got := recordedSince(t, before, "ResumeRetryGiveUp", "failure"); got.count != 0 { + t.Errorf("ResumeRetryGiveUp failures = %d on Aborted, want 0", got.count) + } + }) + + t.Run("rejects negative sweperf_resume_retry_budget", func(t *testing.T) { + if err := knobsCodec.Validate(knobs{ResumeRetryBudget: -1}); err == nil { + t.Error("Validate accepted negative sweperf_resume_retry_budget") + } + }) +} + +func TestNextResumeBackoff(t *testing.T) { + for _, tc := range []struct{ in, want time.Duration }{ + {50 * time.Millisecond, 100 * time.Millisecond}, + {2 * time.Second, 4 * time.Second}, + {3 * time.Second, 4 * time.Second}, + {4 * time.Second, 4 * time.Second}, + } { + if got := nextResumeBackoff(tc.in); got != tc.want { + t.Errorf("nextResumeBackoff(%v) = %v, want %v", tc.in, got, tc.want) + } + } +}