From d43924b37fd7570ad48cb11111a5d57b13580dad Mon Sep 17 00:00:00 2001
From: Fabian Meyer <44942030+dinooo13@users.noreply.github.com>
Date: Fri, 16 Oct 2026 15:37:56 +0200
Subject: [PATCH] CLI: bench-coordinator times the coordinator's own share of
 release-to-paste

The real coordinator and processors, with stand-ins of fixed latency for the
engine, microphone, paste and polish; what is left of the total is the
coordinator's hand-over and its hops between stages.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 CLAUDE.md                                 |   1 +
 Sources/PladderCLI/BenchCoordinator.swift | 326 ++++++++++++++++++++++
 Sources/PladderCLI/main.swift             |   4 +
 docs/ARCHITECTURE.md                      |   1 +
 docs/BENCHMARKS.md                        |  52 ++++
 5 files changed, 384 insertions(+)
 create mode 100644 Sources/PladderCLI/BenchCoordinator.swift

diff --git a/CLAUDE.md b/CLAUDE.md
index 3b70e85..dba4322 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -28,6 +28,7 @@ swift run -c release pladder-cli <audio file> [--process] [--verbose] # transcri
 ./scripts/make-fixtures.sh                    # synthesise benchmark fixtures into bench/fixtures (gitignored)
 swift run -c release pladder-cli bench bench/fixtures            # whole-buffer benchmark
 swift run -c release pladder-cli bench bench/fixtures --paced     # feed at real time, time endUtterance, check identity
+swift run -c release pladder-cli bench-coordinator [--polish] [--live]   # the coordinator's own share of release-to-paste, stand-ins of fixed latency
 swift run -c release pladder-cli polish <text file> [--model apple|s1-mini|s1-mini-8bit]   # one transcript, cold and warm
 swift run -c release pladder-cli polish-set docs/polish-set.json --model s1-mini-8bit    # a polish model over the test set: error rate, exact matches, timings
 ./scripts/overlay-demo.sh [dir] [--style …] [--speed …]  # play every overlay path with stand-ins, record the screen, cut contact sheets
diff --git a/Sources/PladderCLI/BenchCoordinator.swift b/Sources/PladderCLI/BenchCoordinator.swift
new file mode 100644
index 0000000..1c7cebb
--- /dev/null
+++ b/Sources/PladderCLI/BenchCoordinator.swift
@@ -0,0 +1,326 @@
+import Foundation
+import PladderCore
+import PladderSystem
+
+// The coordinator's own share of release-to-paste: "The coordinator's share" in
+// docs/BENCHMARKS.md. Stand-ins of fixed latency for everything the coordinator
+// drives, so what is left of the total is the coordinator.
+enum CoordinatorBench {
+    struct Options {
+        var runs = 21
+        var hold: Duration = .milliseconds(1_500)
+        var engineLatency: Duration = .milliseconds(150)
+        var pasteLatency: Duration = .zero
+        var polish = false
+        var live = false
+    }
+
+    static let polishLatency: Duration = .milliseconds(300)
+
+    static func run(arguments: [String]) async {
+        var options = Options()
+        var rest = arguments[...]
+        func number(_ value: String?) -> Double {
+            guard let value, let number = Double(value), number >= 0 else { usage() }
+            return number
+        }
+        while let argument = rest.popFirst() {
+            switch argument {
+            case "--runs":
+                guard let value = rest.popFirst(), let count = Int(value) else { usage() }
+                options.runs = count
+            case "--hold": options.hold = .milliseconds(Int(number(rest.popFirst()) * 1_000))
+            case "--engine-latency": options.engineLatency = .milliseconds(Int(number(rest.popFirst())))
+            case "--paste-latency": options.pasteLatency = .milliseconds(Int(number(rest.popFirst())))
+            case "--polish": options.polish = true
+            case "--live": options.live = true
+            default: usage()
+            }
+        }
+        guard options.runs >= 2 else {
+            eprint("--runs must be at least 2 (the first run is discarded)")
+            exit(2)
+        }
+        guard options.hold >= .milliseconds(300) else {
+            eprint("--hold must be at least 0.3 s, the coordinator's minimum recording")
+            exit(2)
+        }
+        await run(options)
+    }
+
+    struct Run {
+        var total: Duration
+        var share: Duration
+        var engineExcess: Duration
+    }
+
+    @MainActor
+    static func run(_ options: Options) async {
+        let engine = FixedLatencyEngine(latency: options.engineLatency)
+        let stamps = EventStamps()
+        var settings = DictationSettings(engineID: FixedLatencyEngine.engineID)
+        settings.polishDictations = options.polish
+        settings.liveTranscript = options.live
+        let coordinator = DictationCoordinator(
+            settings: settings,
+            registry: EngineRegistry([
+                .init(id: engine.id, displayName: engine.displayName, detail: "") { engine },
+            ]),
+            capture: SilentCapture(),
+            output: TimedOutput(latency: options.pasteLatency),
+            refiner: options.polish ? FixedLatencyRefiner(latency: polishLatency) : nil,
+            hotkeyMonitor: SilentHotkeyMonitor(),
+            makePipeline: { StandardProcessors.pipeline(for: $0) },
+            onEvent: { stamps.stamp($0) }
+        )
+        coordinator.start()
+        while !coordinator.engineStatus.isReady || coordinator.state != .idle {
+            try? await Task.sleep(for: .milliseconds(10))
+        }
+
+        var overshoots: [Duration] = []
+        for _ in 0..<11 {
+            let started = ContinuousClock.now
+            await uncancellableSleep(options.engineLatency)
+            overshoots.append(ContinuousClock.now - started - options.engineLatency)
+        }
+
+        printBenchHeader("Pladder coordinator benchmark", engine: engine)
+        print(String(format: "load:    %.2f (1-minute average) at start", loadAverage()))
+        print("runs:    \(options.runs), first discarded, median reported")
+        print("hold:    \(format(options.hold)), engine \(format(options.engineLatency)), "
+            + "paste \(format(options.pasteLatency))"
+            + (options.polish ? ", polish \(format(polishLatency))" : "")
+            + (options.live ? ", Live Transcript" : ""))
+        print("timer:   a \(format(options.engineLatency)) stand-in sleep overshoots by \(format(median(overshoots))) "
+            + "(median), which engine excess includes")
+        print("")
+
+        var kept: [Run] = []
+        for attempt in 1...options.runs {
+            await coordinator.hotkeyPressed()
+            guard coordinator.state.isRecording else {
+                eprint("run \(attempt): the press was refused in state \(coordinator.state)")
+                exit(1)
+            }
+            try? await Task.sleep(for: options.hold)
+            coordinator.hotkeyReleased()
+            await coordinator.inFlight?.value
+            guard let insertion = stamps.insertion, let stopped = stamps.stopped, let inserted = stamps.inserted else {
+                eprint("run \(attempt): nothing was inserted (state \(coordinator.state))")
+                exit(1)
+            }
+            stamps.reset()
+            let timing = insertion.timing
+            let total = inserted - stopped
+            let stages = timing.captureStop + timing.engine + timing.processing + timing.insert + (timing.polish ?? .zero)
+            let run = Run(total: total, share: total - stages, engineExcess: timing.engine - options.engineLatency)
+            let tag = attempt == 1 ? " (discarded)" : ""
+            print("run \(attempt): total \(format(run.total)) (stop \(format(timing.captureStop)), "
+                + "process \(format(timing.processing))), share \(format(run.share)), "
+                + "engine excess \(format(run.engineExcess))\(tag)")
+            if attempt > 1 { kept.append(run) }
+            // Lets the release's detached work (the clipboard prepare, the mute restore) end.
+            try? await Task.sleep(for: .milliseconds(100))
+        }
+        coordinator.stop()
+
+        let shares = kept.map(\.share)
+        print("")
+        print("| Hold | Engine | Total (median) | Share (median / p90 / max) | Engine excess (median) |")
+        print("|---:|---:|---:|---:|---:|")
+        print("| \(format(options.hold)) | \(format(options.engineLatency)) "
+            + "| \(format(median(kept.map(\.total)))) "
+            + "| \(format(median(shares))) / \(format(percentile(shares, 0.9))) / \(format(shares.max() ?? .zero)) "
+            + "| \(format(median(kept.map(\.engineExcess)))) |")
+        print(String(format: "load:    %.2f (1-minute average) at end", loadAverage()))
+    }
+
+    private static func median(_ values: [Duration]) -> Duration {
+        percentile(values, 0.5)
+    }
+
+    private static func percentile(_ values: [Duration], _ fraction: Double) -> Duration {
+        let sorted = values.sorted()
+        guard !sorted.isEmpty else { return .zero }
+        let index = Int((Double(sorted.count - 1) * fraction).rounded())
+        return sorted[index]
+    }
+
+    private static func format(_ duration: Duration) -> String {
+        let ms = duration / .milliseconds(1)
+        return ms.magnitude >= 100 ? String(format: "%.0f ms", ms) : String(format: "%.2f ms", ms)
+    }
+}
+
+// MARK: - Stand-ins
+
+private final class EventStamps: @unchecked Sendable {
+    private let lock = NSLock()
+    private var _stopped: ContinuousClock.Instant?
+    private var _inserted: ContinuousClock.Instant?
+    private var _insertion: DictationCoordinator.Insertion?
+    var stopped: ContinuousClock.Instant? { lock.withLock { _stopped } }
+    var inserted: ContinuousClock.Instant? { lock.withLock { _inserted } }
+    var insertion: DictationCoordinator.Insertion? { lock.withLock { _insertion } }
+
+    func stamp(_ event: DictationCoordinator.Event) {
+        let now = ContinuousClock.now
+        lock.withLock {
+            switch event {
+            case .recordingStopped: _stopped = now
+            case .inserted(let insertion):
+                _inserted = now
+                _insertion = insertion
+            default: break
+            }
+        }
+    }
+
+    func reset() {
+        lock.withLock {
+            _stopped = nil
+            _inserted = nil
+            _insertion = nil
+        }
+    }
+}
+
+// Ignores cancellation, as a CoreML pass in flight does. A strict timer: `Task.sleep`
+// overshoots 150 ms by about 8 ms on an M1 even with zero tolerance, this by 0.2 ms.
+private func uncancellableSleep(_ duration: Duration) async {
+    guard duration > .zero else { return }
+    await withCheckedContinuation { (done: CheckedContinuation<Void, Never>) in
+        let timer = DispatchSource.makeTimerSource(flags: .strict, queue: .global(qos: .userInteractive))
+        timer.schedule(deadline: .now() + duration.timeInterval, leeway: .nanoseconds(0))
+        timer.setEventHandler {
+            timer.cancel()
+            done.resume()
+        }
+        timer.resume()
+    }
+}
+
+private actor FixedLatencyEngine: StreamingTranscriptionEngine {
+    static let engineID = EngineID("fixed-latency")
+    nonisolated let id = FixedLatencyEngine.engineID
+    nonisolated let displayName = "Fixed latency (benchmark)"
+    private(set) var status: EngineStatus = .unloaded
+    private let latency: Duration
+    private var slot = UtteranceSlot<Void>()
+
+    // About what a dictation of this length looks like, with a filler for the processors.
+    private static let text = "So um I think we should move the review to Thursday afternoon and send the notes to the whole team before then."
+
+    init(latency: Duration) {
+        self.latency = latency
+    }
+
+    func load() async throws { status = .ready }
+    func unload() async { status = .unloaded }
+
+    func transcribe(_ samples: [Float]) async throws -> Transcript {
+        await uncancellableSleep(latency)
+        return transcript(samples: samples.count)
+    }
+
+    func beginUtterance() async throws -> Utterance {
+        let (utterance, _) = slot.begin()
+        _ = slot.install((), for: utterance)
+        return utterance
+    }
+
+    func feed(_ samples: [Float], to utterance: Utterance) async {}
+
+    func endUtterance(_ utterance: Utterance, tail: [Float]) async throws -> Transcript {
+        guard slot.release(utterance).wasCurrent else { throw TranscriptionError.notLoaded }
+        await uncancellableSleep(latency)
+        return transcript(samples: 0)
+    }
+
+    func abandonUtterance(_ utterance: Utterance) async {
+        slot.release(utterance)
+    }
+
+    func warmPass() async {
+        await uncancellableSleep(latency)
+    }
+
+    func livePass(_ utterance: Utterance) async -> String? {
+        guard slot.session(for: utterance) != nil else { return nil }
+        await uncancellableSleep(latency)
+        return Self.text
+    }
+
+    // An audio duration of zero lets the coordinator fill in what it fed.
+    private func transcript(samples: Int) -> Transcript {
+        Transcript(
+            text: Self.text,
+            audioDuration: Double(samples) / CapturedAudio.sampleRate,
+            processingTime: latency.timeInterval,
+            engineID: id
+        )
+    }
+}
+
+// Silence at the real rate: a drain or a stop returns what a microphone would have
+// captured since the last one.
+private actor SilentCapture: AudioCapture {
+    private var since: ContinuousClock.Instant?
+
+    func start() async throws -> AsyncStream<Float> {
+        since = .now
+        return AsyncStream { _ in }
+    }
+
+    func drain() async -> [Float] {
+        collect()
+    }
+
+    func stop() async -> CapturedAudio {
+        let samples = collect()
+        since = nil
+        return CapturedAudio(samples: samples)
+    }
+
+    func warmUp() async throws {}
+
+    private func collect() -> [Float] {
+        guard let since else { return [] }
+        let now = ContinuousClock.now
+        self.since = now
+        let count = Int((now - since).timeInterval * CapturedAudio.sampleRate)
+        return [Float](repeating: 0, count: count)
+    }
+}
+
+private struct TimedOutput: TextOutput {
+    let latency: Duration
+
+    func insert(_ text: String, submit: Bool) async throws -> InsertResult {
+        await uncancellableSleep(latency)
+        return .pasted
+    }
+
+    func copy(_ text: String) async {}
+}
+
+private struct FixedLatencyRefiner: TranscriptRefiner {
+    let latency: Duration
+
+    func prepare() async {}
+
+    func refine(_ text: String) async -> String? {
+        await uncancellableSleep(latency)
+        return text
+    }
+}
+
+private struct SilentHotkeyMonitor: HotkeyMonitor {
+    func start(chords: [HotkeyRole: Hotkey], submitKey: Hotkey) -> AsyncStream<HotkeyMonitorEvent> {
+        AsyncStream { _ in }
+    }
+
+    func stop() {}
+    func setCancelKeyEnabled(_ enabled: Bool) {}
+}
diff --git a/Sources/PladderCLI/main.swift b/Sources/PladderCLI/main.swift
index a1b9591..367af7f 100644
--- a/Sources/PladderCLI/main.swift
+++ b/Sources/PladderCLI/main.swift
@@ -10,6 +10,7 @@ func usage() -> Never {
            pladder-cli bench <fixtures dir> [--runs N] [--pause S]
            pladder-cli bench <fixtures dir> --paced [--runs N] [--pause S] [--all] [--live]
            pladder-cli bench-process <fixtures dir> [--runs N]
+           pladder-cli bench-coordinator [--runs N] [--hold S] [--engine-latency MS] [--paste-latency MS] [--polish] [--live]
            pladder-cli polish <text file | -> [--model \(PolishModel.cliNames) | --gguf <file>] [--control <line>] [--instructions <file>]
            pladder-cli polish-set <set.json> [--model \(PolishModel.cliNames) | --gguf <file>] [--control <line>]
     """)
@@ -63,6 +64,9 @@ case "bench":
 case "bench-process":
     arguments.removeFirst()
     ProcessorBench.run(arguments: arguments)
+case "bench-coordinator":
+    arguments.removeFirst()
+    await CoordinatorBench.run(arguments: arguments)
 case "polish", "polish-set":
     let command = arguments.removeFirst()
     var options = PolishOptions()
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index d362749..6fda1da 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -479,6 +479,7 @@ Errors and progress go to stderr, so stdout holds only results.
 | `bench <fixtures dir>` | The whole-buffer benchmark. `--runs N` (default 6, first discarded; 11 to settle a result near the noise line), `--pause S` (idle seconds before every run, default 10). |
 | `bench <dir> --paced` | Feeds each fixture in one-second chunks at real time and times `endUtterance`; also transcribes it whole and reports whether the texts are identical. Skips fixtures under 13 s unless `--all`. `--live` adds the Live Transcript pass every 0.5 s and reports its cost. |
 | `bench-process <dir>` | Times the processor pipeline on the fixtures' scripts, with a filler in every sentence and with a spoken question mark too. `--runs N`, default 31. |
+| `bench-coordinator` | Drives the real coordinator and processors with stand-ins of fixed latency for the engine, microphone, paste and polish, and reports what release-to-paste costs beyond them. `--runs N` (default 21, first discarded), `--hold S` (1.5), `--engine-latency MS` (150), `--paste-latency MS` (0), `--polish` (a 300 ms stand-in), `--live`. |
 | `polish <text file \| ->` | Runs a polish model over one transcript, cold and then warm. `--model apple\|s1-mini\|s1-mini-8bit`, `--instructions <file>` (Apple), `--gguf <file>` (any S1-mini-family model), `--control <line>` (S1-mini). |
 | `polish-set <set.json>` | Runs a polish model over a test set after the app's processors: every answer, word error rate per language, exact matches, timings. `--model`, `--gguf` and `--control` as for `polish`; no `--instructions`. |
 
diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md
index 80f07bd..6984249 100644
--- a/docs/BENCHMARKS.md
+++ b/docs/BENCHMARKS.md
@@ -123,6 +123,53 @@ without them. Run it beside a plain `--paced` run and compare the
 `endUtterance` medians: the difference is what a release that lands next to a
 live pass costs.
 
+### The coordinator's share
+
+The engine benchmarks time the engine; the app's log line times a whole
+dictation. What sits between them, the coordinator's own work on the
+release-to-paste path (the feed hand-over, `endRecording`, the hops between
+stages on the main actor), has a number of its own here:
+
+```sh
+swift run -c release pladder-cli bench-coordinator [--runs N] [--hold S] [--engine-latency MS] [--paste-latency MS] [--polish] [--live]
+```
+
+It builds the real `DictationCoordinator` with the app's processors on a
+fresh install's settings, and stand-ins for the rest: a streaming engine whose
+`endUtterance`, warm pass and live pass each take `--engine-latency` (150 ms by
+default) and ignore cancellation, as a CoreML pass does; a microphone that
+hands out silence at the real rate; a paste of `--paste-latency` (0); and with
+`--polish` a 300 ms polish model. Each run presses, holds for `--hold` seconds
+(1.5), releases and waits for the paste. It touches no microphone, pasteboard
+or hotkey, so it can run beside a copy of Pladder in use.
+
+Per run, `total` is `inserted` minus `recordingStopped`, as in the app's log
+line. `share` is the total minus the stages the `inserted` event reports
+(stop, engine, process, polish, paste): the time spent between them.
+`engine excess` is the engine stage minus the stand-in's latency: the feed
+hand-over and the actor hops around `endUtterance`, plus the stand-in timer's
+own overshoot, which the header prints. 21 runs by default, the first
+discarded.
+
+Use it before and after a change to the coordinator, beside the paced bench:
+the paced bench says what the engine costs, this what the coordinator adds.
+
+First numbers, Apple M1, 16 GB, macOS 27.0, October 2026, 21 runs each,
+load average about 2:
+
+| Variant | Total (median) | Share (median / p90 / max) | Engine excess (median) |
+|---|---:|---:|---:|
+| default | 153 ms | 0.02 / 0.02 / 0.03 ms | 0.15 ms |
+| `--polish` | 453 ms | 0.04 / 0.04 / 0.04 ms | 0.15 ms |
+| `--live` | 153 ms | 0.02 / 0.07 / 0.09 ms | 0.14 ms |
+
+The stand-in timer overshot by 0.1 ms, so the coordinator's hand-over and
+hops come to a few hundredths of a millisecond, and the rest of the total
+above the stand-ins' latencies is the processors' 2 to 3 ms. Two 51-run
+passes gave medians of 156 and 153 ms total and 0.05 and 0.02 ms share. A
+release that lands in a warm or live pass waits for it, as in the app; at
+the default 1.5 s hold none does.
+
 ## Baseline
 
 Reference machine: Apple M1, 16 GB, macOS 26.6.2 (25G83), FluidAudio 0.15.6,
@@ -141,6 +188,11 @@ the models and takes far longer; that is a one-time cost and not what this
 number tracks. The weights run on the Neural Engine and are held outside the
 process, so the footprint understates total memory use.
 
+During the benchmarking for #70 the M1 logged `ANECCompile() FAILED (11)` on
+the first run of each new build, and both builds measured about 0.45 s slower
+than the table below. A pass that logs it is not comparable; the table wants
+re-running on a quiet day (numbers to follow).
+
 | Fixture | Audio | Engine (median) | Spread | Realtime | WER |
 |---|---:|---:|---:|---:|---:|
 | 10s | 9.7 s | 0.237 s | 20 % | 41x | 0.0 % |