diff --git a/.changeset/session-input-policy.md b/.changeset/session-input-policy.md new file mode 100644 index 0000000000..1407b644fc --- /dev/null +++ b/.changeset/session-input-policy.md @@ -0,0 +1,7 @@ +--- +'@fluentui-react-native/desktop-driver': patch +'@fluentui-react-native/desktop-test': patch +'@fluentui-react-native/storybook-desktop': patch +--- + +Enforce input policy per WebDriver session and report case-level input mechanism receipts. diff --git a/packages/agentic/desktop-driver/native/PROTOCOL.md b/packages/agentic/desktop-driver/native/PROTOCOL.md index 8e1db1651f..5848f2a3d1 100644 --- a/packages/agentic/desktop-driver/native/PROTOCOL.md +++ b/packages/agentic/desktop-driver/native/PROTOCOL.md @@ -283,6 +283,39 @@ one input turn and refuses a display name the element does not publish exactly once, so a key never reaches a different action than the one the agent named. +### Clear mechanisms + +The mutating `clear` command takes `{ "elementId": "" }`. Helpers that +advertise `strictClear` in `hello.features` and in the `features` map of the +`probe` result also accept an optional `mechanism` of `"semantic"` or +`"keyboard"`, and answer every successful clear with +`{ "mechanism": "semantic" | "keyboard" }`, naming the mechanism that cleared +the element: + +- Without `mechanism`, the helper writes an empty value through the platform + accessibility value (macOS `AXValue`, the Windows UI Automation Value + pattern) and, when the element has no writable value or the write is + refused, falls back to focusing the element and pressing select-all and + Delete. +- `semantic` uses only the value write and never falls back to the keyboard. + An element without a writable value fails `element-not-interactable` with + effect `none` before anything is dispatched, and a broken element is + `stale-element`. A write that was dispatched and failed settles as any other + mutation does: a refusal that proves the write never acted is `none`, any + other failure is `unknown`, and a write that never settles is + `mutation-indeterminate`. +- `keyboard` skips the value write and clears only with the keyboard, behind + the same input authority, session, focus, and physical input lock checks as + the fallback. A check that refuses, such as `input-unavailable`, dispatches + nothing and is `none`. + +Any other `mechanism`, including one that is not a string, fails +`invalid-params` with effect `none`. Node never sends `mechanism` unless the +feature is negotiated, and treats a result without a recognized `mechanism`, +such as the `null` an older helper returns, as not saying which mechanism +cleared. The parameter, result, and feature are additive and do not change the +wire version. + ### Bounded observation Helpers that advertise `boundedObservation` in `hello.features` and in the diff --git a/packages/agentic/desktop-driver/native/macos/README.md b/packages/agentic/desktop-driver/native/macos/README.md index 6a803f8dd3..e333b9eaed 100644 --- a/packages/agentic/desktop-driver/native/macos/README.md +++ b/packages/agentic/desktop-driver/native/macos/README.md @@ -289,6 +289,17 @@ responsible process and an unlocked interactive Aqua session. Missing authority returns `input-unavailable`; the provider never substitutes unrelated accessibility activation and reports physical success. +The helper advertises `strictClear`. `clear` writes an empty `AXValue` when +`AXUIElementIsAttributeSettable` reports the value settable, and otherwise +activates the application, focuses the element, and presses Command-A and +Delete under the physical input lock. A request may name `mechanism`: +`semantic` only writes `AXValue` and fails `element-not-interactable` with +effect `none` when the value is not settable, and `keyboard` skips the write +and needs the same physical input authority as the fallback. Every successful +clear returns `{ "mechanism": "semantic" | "keyboard" }`. The `probe` feature +is `true` even without Accessibility trust, because each mechanism reports its +own refusal. + A locked screen or a login session that is not the active console prevents applications from publishing complete accessibility windows and from receiving physical input. Launch, attach, window discovery, and new pointer, keyboard, diff --git a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Accessibility.swift b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Accessibility.swift index a8e67b97eb..bb9ef8508a 100644 --- a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Accessibility.swift +++ b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Accessibility.swift @@ -378,11 +378,27 @@ final class AccessibilityEngine { } } - func tryClearValue(_ elementID: String) throws -> Bool { + /// Writes an empty `AXValue` and reports whether that cleared the element. Without `strict`, an element whose value + /// cannot be written answers false so the caller can clear with the keyboard. With it, the method never answers + /// false: such an element fails `element-not-interactable` before anything is dispatched, and a failed check or + /// write fails with the effect the ledger recorded. + func tryClearValue(_ elementID: String, strict: Bool = false) throws -> Bool { let record = try requireRecord(elementID) try validateLive(record) var settable: DarwinBoolean = false let settableError = AXUIElementIsAttributeSettable(record.element, kAXValueAttribute as CFString, &settable) + if strict { + if settableError == .attributeUnsupported || settableError == .noValue + || (settableError == .success && !settable.boolValue) + { + try fail(ErrorCode.notInteractable, "The element does not expose a writable value to clear.") + } + try throwAX( + settableError, + operation: "Checking whether the element value can be set", + staleMessage: "Element \"\(elementID)\" is no longer available." + ) + } if settableError == .attributeUnsupported || settableError == .noValue || !settable.boolValue { return false } @@ -398,6 +414,9 @@ final class AccessibilityEngine { if error == .invalidUIElement { try fail(ErrorCode.staleElement, "Element \"\(elementID)\" is no longer available.") } + if strict { + try throwAX(error, operation: "Clearing the element value") + } return error == .success } diff --git a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Driver.swift b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Driver.swift index 0139f02414..da3d893ae4 100644 --- a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Driver.swift +++ b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Driver.swift @@ -198,8 +198,9 @@ final class Driver { ) return CommandResult() case "clear": - try clear(elementID: try requireString(params, "elementId"), token: token) - return CommandResult() + let elementID = try requireString(params, "elementId") + let mechanism = try clearMechanism(params) + return CommandResult(result: ["mechanism": try clear(elementID: elementID, mechanism: mechanism, token: token)]) case "sendKeys": guard let text = params["text"] as? String else { try fail(ErrorCode.invalidParams, "The sendKeys request requires a string \"text\".") @@ -296,6 +297,7 @@ final class Driver { "physicalClick": trusted, "screenshot": screenshot, "setWindowRect": trusted, + "strictClear": true, "wheel": trusted, "windowEvents": trusted, ], @@ -520,16 +522,34 @@ final class Driver { } } - private func clear(elementID: String, token: CancellationToken) throws { + /// The clear mechanism a request names, or nil when the helper may choose. Anything else is refused before dispatch. + private func clearMechanism(_ params: JSONObject) throws -> String? { + guard let value = params["mechanism"] else { + return nil + } + guard let mechanism = value as? String, mechanism == "semantic" || mechanism == "keyboard" else { + try fail(ErrorCode.invalidParams, "The clear mechanism must be \"semantic\" or \"keyboard\".") + } + return mechanism + } + + /// Clears the element and returns the mechanism that did. Without a requested mechanism the value write comes first + /// and the keyboard is its fallback; a requested mechanism is the only one used. + private func clear(elementID: String, mechanism: String?, token: CancellationToken) throws -> String { let window = try windowForElement(elementID) _ = try accessibility.snapshot(elementID: elementID, window: window, token: token) - if try accessibility.tryClearValue(elementID) { - return + if mechanism != "keyboard", try accessibility.tryClearValue(elementID, strict: mechanism == "semantic") { + return "semantic" + } + guard mechanism != "semantic" else { + try fail(ErrorCode.internalError, "The semantic clear ended without clearing or failing.") } guard input.physicalInputAvailable() else { try fail( ErrorCode.inputUnavailable, - "The element does not expose a writable value and physical input requires Accessibility permission." + mechanism == "keyboard" + ? "Physical keyboard input requires Accessibility permission in System Settings > Privacy & Security > Accessibility." + : "The element does not expose a writable value and physical input requires Accessibility permission." ) } try withPhysicalInput(token: token) { @@ -538,6 +558,7 @@ final class Driver { try token.wait(milliseconds: 16) try input.pressCommandADelete(token: token) } + return "keyboard" } private func sendKeys(elementID: String, text: String, token: CancellationToken) throws { diff --git a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Host.swift b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Host.swift index 62a46b3185..bd81f63e6b 100644 --- a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Host.swift +++ b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Host.swift @@ -31,6 +31,7 @@ func createHello() -> JSONObject { "releaseActions", "setWindowRect", "source", + "strictClear", "tree", "wheel", "windowEvents", diff --git a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/SelfTest.swift b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/SelfTest.swift index 4a0d75eb0e..be7174e15e 100644 --- a/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/SelfTest.swift +++ b/packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/SelfTest.swift @@ -772,6 +772,26 @@ final class SelfTestRunner { actions?.code == ErrorCode.invalidParams && actions?.effect == .noEffect, "a malformed input sequence reports no effect" ) + for mechanism: Any in ["pointer", "", 1, NSNull()] { + residue.dispatching() + let clear = failure("clear", ["elementId": "element-missing", "mechanism": mechanism]) as? HelperError + check( + clear?.code == ErrorCode.invalidParams && clear?.effect == .noEffect, + "a clear naming an unknown mechanism (\(mechanism)) is refused before dispatch" + ) + } + for mechanism in ["semantic", "keyboard"] { + let clear = failure("clear", ["elementId": "element-missing", "mechanism": mechanism]) as? HelperError + check( + clear?.code == ErrorCode.staleElement && clear?.effect == .noEffect, + "a \(mechanism) clear of a missing element reports no effect" + ) + } + let probe = try? driver.execute(command: "probe", params: [:], token: token).result as? JSONObject + check( + (probe?["features"] as? JSONObject)?["strictClear"] as? Bool == true, + "probe reports that clear honors a requested mechanism" + ) let read = failure("windows") as? HelperError check(read != nil && read?.effect == nil, "a read-only command does not claim an effect") let unsupported = failure("not-a-command") as? HelperError @@ -931,6 +951,7 @@ final class SelfTestRunner { (hello["features"] as? [String])?.contains("performAccessibilityAction") == true, "hello advertises performing accessibility actions" ) + check((hello["features"] as? [String])?.contains("strictClear") == true, "hello advertises strict clear") } private func testAccessibilityActionNames() { diff --git a/packages/agentic/desktop-driver/native/windows/README.md b/packages/agentic/desktop-driver/native/windows/README.md index 3a16feea72..e130e6fdbe 100644 --- a/packages/agentic/desktop-driver/native/windows/README.md +++ b/packages/agentic/desktop-driver/native/windows/README.md @@ -172,6 +172,19 @@ accessibility clicks. Neither names nor keys enter the native command journal. The `observe` `actions` fact uses cached pattern-availability properties, so listing actions adds no per-node provider calls to the bounded walk. +### Clear mechanisms + +The helper advertises `strictClear`. `clear` writes an empty value through the +UIA Value pattern when the element exposes one that is not read-only, and +otherwise presses Ctrl+A and Delete after focusing the element. A request may +name `mechanism`: `semantic` only writes the value and fails +`element-not-interactable` with effect `none` when the pattern is missing or +read-only, and `keyboard` skips the write and takes the same input authority, +focus check, and physical input lock as the fallback. The value write uses the +bounded mutation worker, effect ledger, and lease-scoped quarantine, so a +`semantic` write that never settles is `mutation-indeterminate`. Every +successful clear returns `{ "mechanism": "semantic" | "keyboard" }`. + ### Window events and identity Window lifecycle comes from out-of-context `SetWinEventHook` notifications @@ -213,7 +226,8 @@ point. A `SendInput` call that injects fewer events than requested fails as `click` mode `auto` falls back to accessibility activation when physical input is unavailable, and an explicit `physical` request reports `input-unavailable`. -Element `sendKeys` and the physical `clear` fallback activate the target window +Element `sendKeys` and the keyboard `clear` (the fallback, or a `keyboard` +mechanism) activate the target window and verify keyboard focus before sending, which is stricter than the macOS `clear`; `performActions` activates only when the foreground belongs to another application. The helper has no `uiAccess` manifest, and the occlusion check diff --git a/packages/agentic/desktop-driver/native/windows/src/automation.cpp b/packages/agentic/desktop-driver/native/windows/src/automation.cpp index 2181d6726f..e75f46fda6 100644 --- a/packages/agentic/desktop-driver/native/windows/src/automation.cpp +++ b/packages/agentic/desktop-driver/native/windows/src/automation.cpp @@ -1422,17 +1422,35 @@ void Automation::PerformAccessibilityAction(const std::string& elementId, const } } -bool Automation::TryClearValue(const std::string& elementId) { +bool Automation::TryClearValue(const std::string& elementId, bool strict) { const ElementRecord& record = RequireRecord(elementId); winrt::com_ptr value; - if (FAILED(record.element->GetCurrentPatternAs(UIA_ValuePatternId, IID_PPV_ARGS(value.put()))) || !value) { - return false; + const HRESULT query = record.element->GetCurrentPatternAs(UIA_ValuePatternId, IID_PPV_ARGS(value.put())); + if (FAILED(query) || !value) { + if (!strict) { + return false; + } + if (query == UIA_E_ELEMENTNOTAVAILABLE) { + Fail(kErrorStaleElement, "The element is no longer available."); + } + if (SUCCEEDED(query) || query == UIA_E_NOTSUPPORTED || query == E_NOINTERFACE || query == E_NOTIMPL || + query == E_FAIL) { + Fail(kErrorNotInteractable, "The element does not expose a writable value to clear."); + } + FailHresult(kErrorAutomationFailed, "Reading the element value pattern", query); } const auto readOnly = std::make_shared(TRUE); const HRESULT state = BoundedRead([value, readOnly]() { return value->get_CurrentIsReadOnly(readOnly.get()); }, "Reading the element value state"); if (FAILED(state) || *readOnly != FALSE) { - return false; + if (!strict) { + return false; + } + if (state == UIA_E_ELEMENTNOTAVAILABLE) { + Fail(kErrorStaleElement, "The element is no longer available."); + } + CheckHresult(kErrorAutomationFailed, "Reading the element value state", state); + Fail(kErrorNotInteractable, "The element's value is read-only, so it cannot be cleared."); } const HRESULT result = CountedMutation( [value]() { @@ -1442,6 +1460,13 @@ bool Automation::TryClearValue(const std::string& elementId) { return assigned; }, "Clearing the element value"); + if (strict) { + if (result == UIA_E_ELEMENTNOTAVAILABLE) { + Fail(kErrorStaleElement, "The element is no longer available."); + } + CheckHresult(result == UIA_E_ELEMENTNOTENABLED ? kErrorNotInteractable : kErrorAutomationFailed, + "Clearing the element value", result); + } return SUCCEEDED(result); } diff --git a/packages/agentic/desktop-driver/native/windows/src/automation.h b/packages/agentic/desktop-driver/native/windows/src/automation.h index 8388fd088d..8f0a5c9565 100644 --- a/packages/agentic/desktop-driver/native/windows/src/automation.h +++ b/packages/agentic/desktop-driver/native/windows/src/automation.h @@ -103,7 +103,13 @@ class Automation { void AccessibilityClick(const std::string& elementId, const CancellationToken& token); json::Value AccessibilityActions(const std::string& elementId, const CancellationToken& token); void PerformAccessibilityAction(const std::string& elementId, const std::string& key, const CancellationToken& token); - bool TryClearValue(const std::string& elementId); + // Writes an empty value through the Value pattern and reports whether that + // cleared the element. Without `strict`, an element without a writable value + // or a refused write returns false so the caller can clear with the + // keyboard. With it, the method never returns false: such an element fails + // as not interactable before anything is dispatched, and a failed write + // fails with the effect the ledger recorded. + bool TryClearValue(const std::string& elementId, bool strict = false); bool HasKeyboardFocus(const std::string& elementId); void ForgetWindow(const std::string& windowId); diff --git a/packages/agentic/desktop-driver/native/windows/src/driver.cpp b/packages/agentic/desktop-driver/native/windows/src/driver.cpp index c88fa2c719..30ce1ebd88 100644 --- a/packages/agentic/desktop-driver/native/windows/src/driver.cpp +++ b/packages/agentic/desktop-driver/native/windows/src/driver.cpp @@ -41,6 +41,19 @@ std::string RequireString(const json::Value& params, std::string_view key) { return value->AsString(); } +// The clear mechanism a request names, or empty when the helper may choose. +// Anything else is refused before dispatch. +std::string RequireClearMechanism(const json::Value& params) { + const json::Value* mechanism = params.Find("mechanism"); + if (mechanism == nullptr) { + return {}; + } + if (!mechanism->IsString() || (mechanism->AsString() != "semantic" && mechanism->AsString() != "keyboard")) { + Fail(kErrorInvalidParams, "The clear mechanism must be \"semantic\" or \"keyboard\"."); + } + return mechanism->AsString(); +} + int ActionButton(const json::Value& action) { const json::Value* button = action.Find("button"); if (button == nullptr || !button->IsNumber()) { @@ -274,6 +287,7 @@ json::Value Driver::ProbeResult(const json::Value& params) { features.Set("physicalClick", json::Value::Bool(physical)); features.Set("screenshot", json::Value::Bool(screenshot)); features.Set("setWindowRect", json::Value::Bool(true)); + features.Set("strictClear", json::Value::Bool(true)); features.Set("wheel", json::Value::Bool(physical)); features.Set("windowEvents", json::Value::Bool(true)); @@ -769,9 +783,14 @@ CommandResult Driver::ExecuteCommand(const std::string& command, const json::Val } if (command == "clear") { const std::string elementId = RequireString(params, "elementId"); + const std::string mechanism = RequireClearMechanism(params); RequireLiveElement(elementId, token); const MutationScope scope(LeaseIdForElement(elementId)); - if (!automation_.TryClearValue(elementId)) { + const bool semantic = mechanism != "keyboard" && automation_.TryClearValue(elementId, mechanism == "semantic"); + if (!semantic && mechanism == "semantic") { + Fail(kErrorInternal, "The semantic clear ended without clearing or failing."); + } + if (!semantic) { RunPhysical(token, [&]() { const HWND window = FocusForKeyboard(elementId, token); const KeyboardGuardScope guard(input_, KeyboardGuardFor(window)); @@ -779,7 +798,8 @@ CommandResult Driver::ExecuteCommand(const std::string& command, const json::Val input_.PressChord({VK_DELETE}, token); }); } - result.result = json::Value::Null(); + result.result = json::Value::Object(); + result.result.Set("mechanism", json::Value::String(std::string(semantic ? "semantic" : "keyboard"))); return result; } if (command == "sendKeys") { diff --git a/packages/agentic/desktop-driver/native/windows/src/host.cpp b/packages/agentic/desktop-driver/native/windows/src/host.cpp index 33438b374d..e57a83c72f 100644 --- a/packages/agentic/desktop-driver/native/windows/src/host.cpp +++ b/packages/agentic/desktop-driver/native/windows/src/host.cpp @@ -51,7 +51,7 @@ json::Value CreateHello() { for (const char* feature : {"accessibilityActions", "accessibilityClick", "performAccessibilityAction", "activeElement", "boundedObservation", "capture", "elementCapture", "find", "hitTest", "keyboard", "launch", "attach", "pointer", "probe", "releaseActions", - "setWindowRect", "source", "tree", "wheel", "windowEvents"}) { + "setWindowRect", "source", "strictClear", "tree", "wheel", "windowEvents"}) { features.Append(json::Value::String(std::string(feature))); } hello.Set("features", features); diff --git a/packages/agentic/desktop-driver/native/windows/src/selftest.cpp b/packages/agentic/desktop-driver/native/windows/src/selftest.cpp index 4dc88d981d..a357f57b6d 100644 --- a/packages/agentic/desktop-driver/native/windows/src/selftest.cpp +++ b/packages/agentic/desktop-driver/native/windows/src/selftest.cpp @@ -404,6 +404,11 @@ void TestHello() { std::any_of(features->Items().begin(), features->Items().end(), [](const json::Value& feature) { return feature.AsString() == "boundedObservation"; }); Check(announcesObservation, "hello announces bounded observation"); + const bool announcesStrictClear = + features != nullptr && + std::any_of(features->Items().begin(), features->Items().end(), + [](const json::Value& feature) { return feature.AsString() == "strictClear"; }); + Check(announcesStrictClear, "hello announces strict clear"); } void TestReleaseLedger() { @@ -2817,6 +2822,50 @@ void TestDriverMutationEffects() { Check(recordedInputs.empty() && buttonClicks.load() == 0 && IsWindow(window) != FALSE, "rejected and cancelled mutations reach nothing"); + // A clear that names its mechanism uses only that mechanism. + const auto clearWith = [](const std::string& elementId, const char* mechanism) { + json::Value request = StringFields({{"elementId", elementId}}); + if (mechanism != nullptr) { + request.Set("mechanism", json::Value::String(std::string(mechanism))); + } + return request; + }; + json::Value numericMechanism = StringFields({{"elementId", editId}}); + numericMechanism.Set("mechanism", json::Value::Integer(1)); + bool unknownMechanismsRefused = true; + for (const json::Value& params : std::vector{clearWith(editId, "pointer"), clearWith(editId, ""), + numericMechanism}) { + unknownMechanismsRefused = + unknownMechanismsRefused && effectOf("clear", params, token, &code) == "none" && code == kErrorInvalidParams; + } + Check(unknownMechanismsRefused, "a clear naming an unknown mechanism is refused before dispatch"); + SetWindowTextW(edit, L"abc"); + Check(effectOf("clear", clearWith(buttonId, "semantic"), token, &code) == "none" && code == kErrorNotInteractable, + "a semantic clear of an element without a value is refused before dispatch"); + SendMessageW(edit, EM_SETREADONLY, TRUE, 0); + Check(effectOf("clear", clearWith(editId, "semantic"), token, &code) == "none" && code == kErrorNotInteractable && + GetWindowTextLengthW(edit) == 3, + "a semantic clear of a read-only value is refused before dispatch"); + SendMessageW(edit, EM_SETREADONLY, FALSE, 0); + driver.InputForTesting().UseTestEnvironment([]() { + return InteractiveWith([](InputEnvironment& environment) { environment.lock = SessionLock::Locked; }); + }); + Check(effectOf("clear", clearWith(editId, "keyboard"), token, &code) == "none" && code == kErrorInputUnavailable && + GetWindowTextLengthW(edit) == 3, + "a keyboard clear honors the input gates instead of writing the value"); + driver.InputForTesting().UseTestEnvironment([]() { return InteractiveEnvironmentForTesting(); }); + const CommandResult semanticClear = driver.Execute("clear", clearWith(editId, "semantic"), token); + Check(semanticClear.result.StringField("mechanism") == "semantic" && GetWindowTextLengthW(edit) == 0, + "a semantic clear writes the value and reports it"); + SetWindowTextW(edit, L"abc"); + const CommandResult chosenClear = driver.Execute("clear", clearWith(editId, nullptr), token); + Check(chosenClear.result.StringField("mechanism") == "semantic" && GetWindowTextLengthW(edit) == 0 && + recordedInputs.empty(), + "a clear that names no mechanism reports the value write it chose"); + const CommandResult probe = driver.Execute("probe", empty, token); + const json::Value* features = probe.result.Find("features"); + Check(features != nullptr && features->BoolField("strictClear"), "the probe reports strict clear"); + // Stopped after part of the command reached the application. recordedInputs.clear(); const json::Value brokenSequence = diff --git a/packages/agentic/desktop-driver/references/protocol.md b/packages/agentic/desktop-driver/references/protocol.md index c8a40be15b..a9570a475b 100644 --- a/packages/agentic/desktop-driver/references/protocol.md +++ b/packages/agentic/desktop-driver/references/protocol.md @@ -39,12 +39,21 @@ Supported extension capabilities: - `furn:renderer`; - `furn:launchMode`: `launch` or `attach`; - `furn:clickMode`: `physical`, `accessibility`, or `auto`; +- `furn:inputMode`: `accessible`, `physical`, or `compatibility` (the default); - returned `furn:features`. Capability matching implements W3C `alwaysMatch` and ordered `firstMatch`, rejects duplicate merge keys, and returns `session not created` when no registered target can satisfy a candidate. +`furn:inputMode` is fixed for the session and returned in its capabilities. +`accessible` permits accessibility clicks, named accessibility actions, and +semantic clears, and rejects keyboard, pointer, and wheel input before +dispatch. `physical` permits physical and accessibility input, but clears only +with keyboard input. `compatibility` preserves the previous behavior and leaves +mechanism choice to the host. A strict mode pins `furn:clickMode`; a conflicting +explicit click mode prevents session creation. + ## W3C routes Implemented standard behavior: @@ -63,6 +72,10 @@ Implemented standard behavior: Implemented `furn` extensions: +- `GET /session/{id}/furn/input`, returning the fixed input mode and counts of + input deliveries by mechanism; +- `POST /session/{id}/element/{element-id}/furn/accessibility-action`, invoking + one uniquely named action published by that element; - `GET /session/{id}/furn/manifest`; - `GET|POST /session/{id}/furn/story`; - `POST /session/{id}/furn/story/reset`; @@ -150,7 +163,9 @@ reads `AXUIElementCopyActionNames` and lists an authored custom action, such as `Toggle` or `Select`, by name without any provider handle, alongside any standard actions the element publishes. An element that publishes none returns `[]`; hosts without the feature, including Windows and Win32, return `unsupported operation` -rather than `[]`. The property only lists actions; it does not invoke them. +rather than `[]`. The property only lists actions; it does not invoke them. The named-action +extension invokes one action by its unique published name; its request is +serialized with input and obeys the session's `furn:inputMode`. What an element publishes depends on the renderer: React Native macOS Fabric publishes every declared `accessibilityActions` entry by `name`, while Paper publishes only entries that declare a `label` and lists them by that label diff --git a/packages/agentic/desktop-driver/references/test-integration.md b/packages/agentic/desktop-driver/references/test-integration.md index 6e09fb1c46..b1cb942a58 100644 --- a/packages/agentic/desktop-driver/references/test-integration.md +++ b/packages/agentic/desktop-driver/references/test-integration.md @@ -38,6 +38,7 @@ import { connectDesktopWebdriver } from '@fluentui-react-native/desktop-driver/w const desktop = await connectDesktopWebdriver({ clickMode: 'auto', + inputMode: 'accessible', launchMode: 'attach', platformName: 'macos', targetId: 'component-storybook-macos', @@ -53,6 +54,21 @@ try { } ``` +`inputMode` defaults to `compatibility`. Use `accessible` to refuse keyboard, +pointer, and wheel input before dispatch, or `physical` to require physical +clears and clicks while still permitting accessibility actions. The selected +mode is fixed in the session's returned `furn:inputMode` capability. Read +`desktop.input()` (or `desktop.browser.desktopInput()`) for that mode and the +session's delivery counts. A declared test runner compares those receipts with +the case's `requires`; an undeclared mechanism fails the test, while an unused +requirement is reported without failing it. + +WDIO's `element.desktopPerformAccessibilityAction(name)` invokes a uniquely +published accessibility action without pointer or keyboard input. Declare +`perform-accessibility-action` on the case that uses it. In an accessible +session, `setValue` is rejected before WDIO's client-side clear step; `clear` +remains available as a semantic clear. + Use standard WebdriverIO APIs only where the [implemented protocol](protocol.md) supports them. Browser navigation, cookies, frames, JavaScript execution, prompts, printing, CSS values, and shadow roots @@ -111,11 +127,12 @@ package. Static plans below are unchanged. The descriptor that `desktopIt` accepts (`requires`, `input`, `platforms`, `scope`, and declared gaps) is defined and decided in `./authoring`, so Storybook -and `desktop-test` share one implementation. A runner that executes one input lane -pins `furn:clickMode` with `resolveDesktopClickMode`: `element.click()` is then an -accessibility click in the accessible lane and a physical click in the physical -lane. `missingDesktopRequirements` judges a case's `requires` against the -session's `furn:features`, and judges a `click` requirement by the pinned mode. +and `desktop-test` share one implementation. A runner that executes one input +lane requests and verifies the matching `furn:inputMode`; the policy pins +`furn:clickMode`, so `element.click()` is an accessibility click in the +accessible lane and a physical click in the physical lane. +`missingDesktopRequirements` judges a case's `requires` against the session's +`furn:features`, and judges a `click` requirement by the pinned mode. ## Author portable story plans diff --git a/packages/agentic/desktop-driver/src/authoring/index.ts b/packages/agentic/desktop-driver/src/authoring/index.ts index 96cae618c8..9c25e6a6e3 100644 --- a/packages/agentic/desktop-driver/src/authoring/index.ts +++ b/packages/agentic/desktop-driver/src/authoring/index.ts @@ -16,27 +16,40 @@ export { } from './storyTests.js'; export { accessibleLaneMaskedFeatures, + compareDesktopInputReceipts, + desktopInputMechanisms, + desktopInputModeAllows, + desktopInputModes, desktopInputLanes, desktopRequirements, + describeUndeclaredDesktopInput, matchesDesktopPlatforms, matchesDesktopScope, mergeDesktopCaseOptions, resolveDesktopInput, resolveDesktopScopeDomains, resolveDesktopScopeValues, + subtractDesktopInputReceipts, unmaskedAccessibleFeatures, validateDesktopCaseOptions, validateDesktopGroupOptions, + validateDesktopInputReceipts, + validateDesktopSessionInput, } from './input.js'; export type { AccessibleLaneMaskedFeature, DesktopCaseOptions, DesktopInput, + DesktopInputComparison, DesktopInputLane, + DesktopInputMechanism, + DesktopInputMode, + DesktopInputReceipts, DesktopPlatforms, DesktopRequirement, DesktopScopeDomains, DesktopScopeValues, + DesktopSessionInput, ResolvedDesktopCaseOptions, } from './input.js'; export { decideDesktopCase, missingDesktopRequirements, resolveDesktopClickMode } from './selection.js'; diff --git a/packages/agentic/desktop-driver/src/authoring/input.test.ts b/packages/agentic/desktop-driver/src/authoring/input.test.ts index 0077d51cfc..3023b25cfa 100644 --- a/packages/agentic/desktop-driver/src/authoring/input.test.ts +++ b/packages/agentic/desktop-driver/src/authoring/input.test.ts @@ -1,14 +1,19 @@ import { accessibleLaneMaskedFeatures, + compareDesktopInputReceipts, + describeUndeclaredDesktopInput, + desktopInputModeAllows, matchesDesktopPlatforms, matchesDesktopScope, mergeDesktopCaseOptions, resolveDesktopInput, resolveDesktopScopeDomains, resolveDesktopScopeValues, + subtractDesktopInputReceipts, unmaskedAccessibleFeatures, validateDesktopCaseOptions, validateDesktopGroupOptions, + validateDesktopSessionInput, } from './input.js'; describe('accessible lane masks', () => { @@ -98,10 +103,17 @@ describe('desktop input descriptors', () => { test('rejects mixed transports and explicit widening', () => { expect(() => resolveDesktopInput(undefined, ['keyboard', 'accessibility-click'])).toThrow('split the case'); + expect(() => resolveDesktopInput('both', ['physical-click', 'perform-accessibility-action'])).toThrow('split the case'); expect(() => resolveDesktopInput('both', ['physical-click'])).toThrow('cannot widen'); expect(() => resolveDesktopInput('physical', ['accessibility-click'])).toThrow('cannot widen'); }); + test('runs a case that needs both transports only in a declared physical lane', () => { + expect(resolveDesktopInput('physical', ['physical-click', 'perform-accessibility-action'])).toBe('physical'); + expect(resolveDesktopInput('physical', ['keyboard', 'accessibility-click'])).toBe('physical'); + expect(resolveDesktopInput(undefined, ['perform-accessibility-action'])).toBe('accessible'); + }); + test('validates platform filters and negation', () => { expect(matchesDesktopPlatforms(['!win32'], 'windows')).toBe(true); expect(matchesDesktopPlatforms(['!win32'], 'win32')).toBe(false); @@ -153,3 +165,61 @@ describe('desktop scope domains', () => { expect(() => matchesDesktopScope(inherited, supplied, domains)).toThrow('unknown scope key'); }); }); + +describe('desktop input receipts', () => { + const kase = (options: object) => validateDesktopCaseOptions(options); + + test('fails a mechanism nothing declared and reports a declared transport left unused', () => { + expect(compareDesktopInputReceipts(kase({ requires: ['click'] }), { 'accessibility-click': 2 })).toEqual({ + undeclared: [], + unused: [], + }); + expect(compareDesktopInputReceipts(kase({ requires: ['physical-click', 'keyboard'] }), { pointer: 1 })).toEqual({ + undeclared: [], + unused: ['keyboard'], + }); + expect(compareDesktopInputReceipts(kase({ requires: ['accessibility-click'] }), { 'physical-click': 1, 'semantic-clear': 1 })).toEqual({ + undeclared: ['physical-click'], + unused: ['accessibility-click'], + }); + expect(compareDesktopInputReceipts(undefined, { 'unattributed-clear': 1, wheel: 1 })).toEqual({ + undeclared: ['unattributed-clear', 'wheel'], + unused: [], + }); + expect(compareDesktopInputReceipts(kase({ requires: ['focus'] }), {})).toEqual({ undeclared: [], unused: [] }); + }); + + test('lets any click declaration account for the transport an auto session chose', () => { + const physical = kase({ requires: ['accessibility-click', 'keyboard'], input: 'physical' }); + expect(compareDesktopInputReceipts(physical, { 'physical-click': 1, keyboard: 1 }, 'auto')).toEqual({ undeclared: [], unused: [] }); + expect(compareDesktopInputReceipts(physical, { 'physical-click': 1, keyboard: 1 }, 'physical')).toEqual({ + undeclared: ['physical-click'], + unused: ['accessibility-click'], + }); + // A pointer action is physical whatever the click mode, so only physical-click declares it. + expect(compareDesktopInputReceipts(kase({ requires: ['click'] }), { pointer: 1 }, 'auto').undeclared).toEqual(['pointer']); + expect(describeUndeclaredDesktopInput(['keyboard', 'accessibility-click'])).toBe( + 'The test delivered input it does not declare: keyboard (declare keyboard), accessibility-click (declare accessibility-click or click).', + ); + }); + + test('takes per-case deltas from a shared session and rejects malformed reports', () => { + expect(subtractDesktopInputReceipts({ keyboard: 3, 'physical-click': 1 }, { keyboard: 1, 'physical-click': 1 })).toEqual({ + keyboard: 2, + }); + expect(() => compareDesktopInputReceipts(undefined, { typing: 1 })).toThrow('unknown mechanism'); + expect(() => compareDesktopInputReceipts(undefined, { keyboard: 0 })).toThrow('positive integer'); + expect(validateDesktopSessionInput({ mode: 'accessible', receipts: {} })).toEqual({ mode: 'accessible', receipts: {} }); + expect(() => validateDesktopSessionInput({ receipts: {} })).toThrow('mode'); + }); + + test('lets each policy deliver only its own mechanisms', () => { + expect(desktopInputModeAllows('accessible', 'accessibility-action')).toBe(true); + expect(desktopInputModeAllows('accessible', 'semantic-clear')).toBe(true); + expect(desktopInputModeAllows('accessible', 'pointer')).toBe(false); + expect(desktopInputModeAllows('accessible', 'unattributed-clear')).toBe(false); + expect(desktopInputModeAllows('physical', 'accessibility-click')).toBe(true); + expect(desktopInputModeAllows('physical', 'semantic-clear')).toBe(false); + expect(desktopInputModeAllows('compatibility', 'unattributed-clear')).toBe(true); + }); +}); diff --git a/packages/agentic/desktop-driver/src/authoring/input.ts b/packages/agentic/desktop-driver/src/authoring/input.ts index b81575801e..4bf362d7a3 100644 --- a/packages/agentic/desktop-driver/src/authoring/input.ts +++ b/packages/agentic/desktop-driver/src/authoring/input.ts @@ -7,6 +7,7 @@ export const desktopRequirements = [ 'element-screenshot', 'focus', 'keyboard', + 'perform-accessibility-action', 'physical-click', 'screenshot', 'set-window-rect', @@ -18,6 +19,142 @@ export type DesktopInput = 'accessible' | 'physical' | 'both'; export type DesktopInputLane = Exclude; export const desktopInputLanes = ['accessible', 'physical'] as const satisfies readonly DesktopInputLane[]; +/** + * The input policy a Desktop Driver session enforces, fixed when the session is created (`furn:inputMode`). `accessible` + * delivers only semantic input, `physical` clicks physically and clears with the keyboard, and `compatibility`, the + * default, enforces nothing beyond the host's own masks. + */ +export const desktopInputModes = ['accessible', 'compatibility', 'physical'] as const; +export type DesktopInputMode = (typeof desktopInputModes)[number]; + +/** + * How a session delivered input, as it records it when it dispatches. `unattributed-clear` is a clear whose host did not + * say whether it set the value or typed, so it counts as possible keyboard input. + */ +export const desktopInputMechanisms = [ + 'accessibility-action', + 'accessibility-click', + 'keyboard', + 'physical-click', + 'pointer', + 'semantic-clear', + 'unattributed-clear', + 'wheel', +] as const; +export type DesktopInputMechanism = (typeof desktopInputMechanisms)[number]; + +/** How many times a session delivered input by each mechanism. A mechanism it never used is absent. */ +export type DesktopInputReceipts = Readonly>>; + +/** What `GET /session/:id/furn/input` returns: the session's policy and the input it delivered. */ +export type DesktopSessionInput = { + mode: DesktopInputMode; + receipts: DesktopInputReceipts; +}; + +/** + * Whether a session's input policy lets it deliver a mechanism. An accessible session delivers only semantic input; a + * physical session clicks physically or semantically but clears only with the keyboard. A strict session never makes an + * unattributed clear, because it always says how to clear. + */ +export function desktopInputModeAllows(mode: DesktopInputMode, mechanism: DesktopInputMechanism): boolean { + switch (mode) { + case 'accessible': + return mechanism === 'accessibility-action' || mechanism === 'accessibility-click' || mechanism === 'semantic-clear'; + case 'physical': + return mechanism !== 'semantic-clear' && mechanism !== 'unattributed-clear'; + default: + return true; + } +} + +/** + * The declarations that account for each mechanism. A semantic clear needs none: it is not physical input, and a case + * that clears declares the keyboard its physical variant types with. + */ +const mechanismRequirements: Readonly> = { + 'accessibility-action': ['perform-accessibility-action'], + 'accessibility-click': ['accessibility-click', 'click'], + keyboard: ['keyboard'], + 'physical-click': ['physical-click', 'click'], + pointer: ['physical-click'], + 'semantic-clear': [], + 'unattributed-clear': ['keyboard'], + wheel: ['wheel'], +}; + +/** What one case delivered, judged against what it declared. */ +export type DesktopInputComparison = { + /** Mechanisms the case delivered that none of its declarations accounts for. Any entry fails the case. */ + undeclared: DesktopInputMechanism[]; + /** Transport requirements the case declared and never exercised. Reported, never failed. */ + unused: DesktopRequirement[]; +}; + +/** Subtracts one receipt snapshot from a later one of the same session, for a runner that shares a session across cases. */ +export function subtractDesktopInputReceipts(after: DesktopInputReceipts, before: DesktopInputReceipts): DesktopInputReceipts { + const delta: Partial> = {}; + for (const mechanism of desktopInputMechanisms) { + const count = (after[mechanism] ?? 0) - (before[mechanism] ?? 0); + if (count > 0) delta[mechanism] = count; + } + return delta; +} + +/** + * Compares the input a case delivered with its declaration. Receipts are the authority: a declaration only promises, so + * a mechanism nothing declared fails the case, while a declared transport the case did not use is only reported. Pass the + * session's click mode: under `auto` the host, not the case, chose how to click, so any click declaration accounts for + * either click transport. + */ +export function compareDesktopInputReceipts( + descriptor: Pick | undefined, + receipts: unknown, + clickMode?: 'accessibility' | 'auto' | 'physical', +): DesktopInputComparison { + const delivered = validateDesktopInputReceipts(receipts); + const used = desktopInputMechanisms.filter((mechanism) => delivered[mechanism] !== undefined); + const declared = descriptor?.requires ?? []; + const hostClicks = clickMode === 'auto' && declared.some((requirement) => clickRequirements.has(requirement)); + const accounts = (mechanism: DesktopInputMechanism): readonly DesktopRequirement[] => + hostClicks && clickMechanisms.has(mechanism) + ? [...mechanismRequirements[mechanism], ...clickRequirements] + : mechanismRequirements[mechanism]; + const undeclared = used.filter( + (mechanism) => accounts(mechanism).length > 0 && !accounts(mechanism).some((requirement) => declared.includes(requirement)), + ); + const unused = declared.filter( + (requirement) => inputRequirements.has(requirement) && !used.some((mechanism) => accounts(mechanism).includes(requirement)), + ); + return { undeclared, unused }; +} + +/** Names each undeclared mechanism with the requirements that would declare it, for the failure of a case that delivered it. */ +export function describeUndeclaredDesktopInput(undeclared: readonly DesktopInputMechanism[]): string { + const named = undeclared.map((mechanism) => `${mechanism} (declare ${mechanismRequirements[mechanism].join(' or ')})`); + return `The test delivered input it does not declare: ${named.join(', ')}.`; +} + +/** Checks a receipt report read from a driver, so a malformed one is an error rather than an empty record. */ +export function validateDesktopInputReceipts(value: unknown, source = 'input receipts'): DesktopInputReceipts { + const receipts = object(value, source); + for (const [mechanism, count] of Object.entries(receipts)) { + if (!(desktopInputMechanisms as readonly string[]).includes(mechanism)) + throw new TypeError(`${source} has unknown mechanism "${mechanism}".`); + if (!Number.isSafeInteger(count) || (count as number) < 1) throw new TypeError(`${source}.${mechanism} must be a positive integer.`); + } + return receipts as DesktopInputReceipts; +} + +/** Checks a session's input report, so a driver that omits its policy is never mistaken for one that enforces it. */ +export function validateDesktopSessionInput(value: unknown, source = 'session input'): DesktopSessionInput { + const report = object(value, source); + exactKeys(report, ['mode', 'receipts'], source); + if (!(desktopInputModes as readonly unknown[]).includes(report.mode)) + throw new TypeError(`${source}.mode must be one of: ${desktopInputModes.join(', ')}.`); + return { mode: report.mode as DesktopInputMode, receipts: validateDesktopInputReceipts(report.receipts, `${source}.receipts`) }; +} + /** * The input an accessible lane keeps from reaching the application. A host that reports any of these as anything other * than `false` cannot back the lane, because a case could then deliver physical input without anyone noticing. @@ -56,15 +193,26 @@ export type ResolvedDesktopCaseOptions = DesktopCaseOptions & { }; const physicalRequirements = new Set(['physical-click', 'keyboard', 'wheel']); -const accessibleRequirements = new Set(['accessibility-click']); +const accessibleRequirements = new Set(['accessibility-click', 'perform-accessibility-action']); +/** Requirements that promise input, so receipts can show one went unused. */ +const inputRequirements = new Set(['click', ...physicalRequirements, ...accessibleRequirements]); +const clickRequirements = new Set(['accessibility-click', 'click', 'physical-click']); +const clickMechanisms = new Set(['accessibility-click', 'physical-click']); const endpoints = ['macos', 'windows', 'win32'] as const; const requirementSet = new Set(desktopRequirements); +/** + * Derives a case's input lane from what it requires. A case that needs both transports runs only in the physical lane, + * whose sessions also deliver declared accessible input, and must say so with an explicit `input: 'physical'`. + */ export function resolveDesktopInput(input?: DesktopInput, requires: readonly DesktopRequirement[] = []): DesktopInput { const physical = requires.some((requirement) => physicalRequirements.has(requirement)); const accessible = requires.some((requirement) => accessibleRequirements.has(requirement)); if (physical && accessible) { - throw new TypeError('A desktop test cannot require both physical and accessible input; split the case.'); + if (input === 'physical') return input; + throw new TypeError( + 'A desktop test that requires both physical and accessible input must declare input: "physical", whose sessions deliver both; otherwise split the case.', + ); } const derived = physical ? 'physical' : accessible ? 'accessible' : 'both'; if (input !== undefined && derived !== 'both' && input !== derived) { diff --git a/packages/agentic/desktop-driver/src/authoring/selection.test.ts b/packages/agentic/desktop-driver/src/authoring/selection.test.ts index 2a8505dafc..ecccdeabff 100644 --- a/packages/agentic/desktop-driver/src/authoring/selection.test.ts +++ b/packages/agentic/desktop-driver/src/authoring/selection.test.ts @@ -70,6 +70,8 @@ describe('click mode and requirement preflight', () => { 'accessibility', ); expect(resolveDesktopClickMode(validateDesktopCaseOptions({ input: 'accessible' }), 'physical', undefined)).toBe('accessibility'); + const mixed = validateDesktopCaseOptions({ input: 'physical', requires: ['physical-click', 'accessibility-click'] }); + expect(resolveDesktopClickMode(mixed, 'auto', undefined)).toBe('auto'); // A physical case must not pin physical click, or a host that masks it fails the session instead of reporting unsupported. expect(resolveDesktopClickMode(validateDesktopCaseOptions({ requires: ['physical-click'] }), 'auto', undefined)).toBe('auto'); expect(resolveDesktopClickMode(validateDesktopCaseOptions({ requires: ['keyboard'] }), 'auto', undefined)).toBe('auto'); diff --git a/packages/agentic/desktop-driver/src/authoring/selection.ts b/packages/agentic/desktop-driver/src/authoring/selection.ts index 383ce5896f..f91c005d33 100644 --- a/packages/agentic/desktop-driver/src/authoring/selection.ts +++ b/packages/agentic/desktop-driver/src/authoring/selection.ts @@ -43,7 +43,7 @@ export type DesktopCaseClickMode = 'auto' | 'physical' | 'accessibility'; /** * The click mode a case runs under. A lane decides it outright, so a `click` requirement is an accessible click in the * accessible lane and a physical click in the physical lane. Without a lane only an accessible case pins its transport, - * so it never moves the pointer. Any other case keeps the configured mode: a physical case on a host that masks physical + * so it never moves the pointer; a case that also requires physical input is a physical case. Any other case keeps the configured mode: a physical case on a host that masks physical * click is then reported unsupported by the requirement preflight, as the skip it replaces was, instead of failing to * create its session. */ @@ -54,7 +54,7 @@ export function resolveDesktopClickMode( ): DesktopCaseClickMode { if (lane === 'accessible') return 'accessibility'; if (lane === 'physical') return 'physical'; - if (descriptor?.input === 'accessible' || descriptor?.requires?.includes('accessibility-click')) return 'accessibility'; + if (descriptor?.input === 'accessible') return 'accessibility'; return configured; } @@ -89,6 +89,7 @@ const requirementFeature: Record, string> = 'element-screenshot': 'elementScreenshot', focus: 'focus', keyboard: 'keyboard', + 'perform-accessibility-action': 'performAccessibilityAction', 'physical-click': 'physicalClick', screenshot: 'screenshot', 'set-window-rect': 'setWindowRect', diff --git a/packages/agentic/desktop-driver/src/client/DesktopDriverClient.ts b/packages/agentic/desktop-driver/src/client/DesktopDriverClient.ts index 17f8b1e67a..33b4ad8066 100644 --- a/packages/agentic/desktop-driver/src/client/DesktopDriverClient.ts +++ b/packages/agentic/desktop-driver/src/client/DesktopDriverClient.ts @@ -1,5 +1,7 @@ import { randomUUID } from 'node:crypto'; +import { validateDesktopSessionInput } from '../authoring/input.js'; +import type { DesktopSessionInput } from '../authoring/input.js'; import { WebDriverError } from '../protocol/errors.js'; import { webElementIdentifier } from '../protocol/constants.js'; import type { DesktopTreeNode, DesktopWindowEventLog } from '../host/types.js'; @@ -198,6 +200,11 @@ export class DesktopSessionClient { return this.command('GET', '/furn/window-events'); } + /** The session's input policy, and how many times it has delivered input through each mechanism. */ + async getInput(): Promise { + return validateDesktopSessionInput(await this.command('GET', '/furn/input')); + } + /** The agent operations this session offers, with the JSON Schema of each one's input and result. */ operations(): Promise { return this.command('GET', '/furn/ops'); @@ -338,6 +345,11 @@ export class DesktopElementClient { return this.command('POST', '/value', { text }); } + /** Performs the one accessibility action the element publishes under this name, such as `increment`. */ + performAccessibilityAction(name: string): Promise { + return this.command('POST', '/furn/accessibility-action', { name }); + } + getText(): Promise { return this.command('GET', '/text'); } diff --git a/packages/agentic/desktop-driver/src/host/types.ts b/packages/agentic/desktop-driver/src/host/types.ts index d7a4eabb43..13b2541ed7 100644 --- a/packages/agentic/desktop-driver/src/host/types.ts +++ b/packages/agentic/desktop-driver/src/host/types.ts @@ -11,6 +11,9 @@ export type Rect = { export type SupportedValue = { supported: true; value: T } | { supported: false; reason: string }; +/** How a host clears a value: by setting it through the accessibility tree, or by selecting and deleting with keys. */ +export type DesktopClearMechanism = 'keyboard' | 'semantic'; + export type DesktopHostFeatures = { /** The host lists an element's accessibility actions, including authored custom actions such as `Select` and `Toggle`. */ accessibilityActions?: boolean; @@ -28,6 +31,11 @@ export type DesktopHostFeatures = { physicalClick: boolean; screenshot: boolean; setWindowRect: boolean; + /** + * The host clears with the mechanism it is asked for: `semantic` only sets the value through the accessibility tree, + * `keyboard` only selects and deletes with synthesized keys, and either fails without falling back to the other. + */ + strictClear?: boolean; wheel: boolean; /** The host records a native window-event timeline for diagnostics. */ windowEvents?: boolean; @@ -277,7 +285,12 @@ export interface DesktopHost { hitTest(windowId: string, x: number, y: number, signal?: AbortSignal): Promise; click(elementId: string, mode: DesktopClickMode, signal?: AbortSignal): Promise; - clear(elementId: string, signal?: AbortSignal): Promise; + /** + * Clears an element's value. Without a mechanism the host chooses, preferring a semantic clear; with one, a host that + * advertises `features.strictClear` uses only that mechanism. Resolves with the mechanism used, or nothing when the host + * cannot say, which counts as possible keyboard input. + */ + clear(elementId: string, signal?: AbortSignal, mechanism?: DesktopClearMechanism): Promise; sendKeys(elementId: string, text: string, signal?: AbortSignal): Promise; performActions(actions: readonly NativeActionSequence[], signal?: AbortSignal): Promise; releaseActions(signal?: AbortSignal): Promise; diff --git a/packages/agentic/desktop-driver/src/hosts/fake/FakeDesktopHost.ts b/packages/agentic/desktop-driver/src/hosts/fake/FakeDesktopHost.ts index eab73c3adf..4fe6fc55d8 100644 --- a/packages/agentic/desktop-driver/src/hosts/fake/FakeDesktopHost.ts +++ b/packages/agentic/desktop-driver/src/hosts/fake/FakeDesktopHost.ts @@ -2,6 +2,7 @@ import { createHash, randomUUID } from 'node:crypto'; import type { ApplicationLease, + DesktopClearMechanism, DesktopHost, DesktopHostEvent, DesktopHostFeatures, @@ -120,6 +121,7 @@ const defaultFeatures: DesktopHostFeatures = { physicalClick: true, screenshot: true, setWindowRect: true, + strictClear: true, wheel: true, }; @@ -535,13 +537,23 @@ export class FakeDesktopHost implements DesktopHost { }); } - async clear(elementId: string, signal?: AbortSignal): Promise { + /** Clears semantically when the element has a value to set, and otherwise with the keyboard, unless told which. */ + async clear(elementId: string, signal?: AbortSignal, mechanism?: DesktopClearMechanism): Promise { return this.mutate('clear', signal, (observed) => { throwIfAborted(observed); const element = this.requireElement(elementId); + const settable = element.value !== undefined; + const used = mechanism ?? (settable ? 'semantic' : 'keyboard'); + if (used === 'semantic' && !settable) { + throw reportEffect(new HostWebDriverError('element not interactable', `Element "${elementId}" has no value to set.`), 'none'); + } + if (used === 'keyboard' && !this.features.keyboard) { + throw reportEffect(new HostUnsupportedError('The fake target does not type.'), 'none'); + } element.value = ''; element.text = ''; - this.actions.push({ type: 'clear', elementId }); + this.actions.push({ type: 'clear', elementId, mechanism: used }); + return used; }); } diff --git a/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.input.test.ts b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.input.test.ts new file mode 100644 index 0000000000..dcc214dc20 --- /dev/null +++ b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.input.test.ts @@ -0,0 +1,130 @@ +import type { DesktopHostFeatures } from '../../host/types'; +import type { NativeDriverArtifact } from '../../native/types'; +import { readFailureEffect } from '../../protocol/settlement'; +import { scriptNativeHelper } from '../../testing/scriptedNativeHelper'; +import type { ScriptedNativeRequest } from '../../testing/scriptedNativeHelper'; +import { FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES, NativeDesktopHost } from './NativeDesktopHost'; +import type { NativeDisabledInputFeature } from './NativeDesktopHost'; + +const mockSpawn = jest.fn(); +jest.mock('node:child_process', () => ({ spawn: (...args: unknown[]) => mockSpawn(...args) })); + +const artifact: NativeDriverArtifact = { + architecture: 'arm64', + artifactId: 'artifact', + artifactRoot: 'artifact-root', + buildFingerprint: 'build', + buildId: 'build', + compatibilityKey: 'compatibility', + configuration: 'release', + endpoints: ['macos'], + executablePath: 'driver', + features: [], + origin: 'cache', + provider: 'macos', + schemaVersion: 1, + signing: { mode: 'none' }, + sourceDigest: 'source', + wireProtocol: { major: 1, minor: 0 }, +}; + +const features: DesktopHostFeatures = { + accessibilityClick: true, + elementScreenshot: true, + focus: true, + keyboard: true, + physicalClick: true, + screenshot: true, + setWindowRect: true, + wheel: true, +}; + +/** A probed host whose helper clears with the mechanism it was asked for, or reports none when it predates strict clear. */ +async function probedHost(strictClear: boolean, disabledInputFeatures: readonly NativeDisabledInputFeature[] = []) { + const received = scriptNativeHelper(mockSpawn, (message: ScriptedNativeRequest, send) => { + const params = message.params as { mechanism?: string } | undefined; + const result = + message.command === 'probe' + ? { + endpoint: 'macos', + features: { ...features, ...(strictClear ? { strictClear } : {}) }, + platformName: 'macos', + protocolVersion: 1, + } + : message.command === 'clear' + ? strictClear + ? { mechanism: params?.mechanism ?? 'semantic' } + : {} + : {}; + send({ id: message.id, result, type: 'response' }); + }); + const host = new NativeDesktopHost({ application: { windowTitle: 'Test App' }, artifact, disabledInputFeatures, endpoint: 'macos' }); + await host.probe(); + return { host, received }; +} + +describe('NativeDesktopHost input delivery', () => { + beforeEach(() => { + jest.replaceProperty(process, 'env', { ...process.env, [FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES]: '' }); + mockSpawn.mockReset(); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + test('asks a strict helper for the mechanism, and returns the one it used', async () => { + const { host, received } = await probedHost(true); + + await expect(host.clear('e1', undefined, 'keyboard')).resolves.toBe('keyboard'); + await expect(host.clear('e1')).resolves.toBe('semantic'); + + expect(received.filter((message) => message.command === 'clear')).toMatchObject([ + { params: { elementId: 'e1', mechanism: 'keyboard' } }, + { params: { elementId: 'e1' } }, + ]); + expect(received[2].params).not.toHaveProperty('mechanism'); + await host.dispose(); + }); + + test('clears semantically where the keyboard is masked, and only when the helper can promise to', async () => { + const strict = await probedHost(true, ['keyboard', 'physicalClick', 'wheel']); + await expect(strict.host.clear('e1')).resolves.toBe('semantic'); + const keyboard = await strict.host.clear('e1', undefined, 'keyboard').catch((error: unknown) => error); + expect(keyboard).toMatchObject({ name: 'HostUnsupportedError' }); + expect(readFailureEffect(keyboard)).toBe('none'); + expect(strict.received.filter((message) => message.command === 'clear')).toMatchObject([ + { params: { elementId: 'e1', mechanism: 'semantic' } }, + ]); + await strict.host.dispose(); + + const legacy = await probedHost(false, ['keyboard', 'physicalClick', 'wheel']); + for (const attempt of [legacy.host.clear('e1'), legacy.host.clear('e1', undefined, 'semantic')]) { + const failure = await attempt.catch((error: unknown) => error); + expect(failure).toMatchObject({ name: 'HostUnsupportedError' }); + expect(readFailureEffect(failure)).toBe('none'); + } + expect(legacy.received.some((message) => message.command === 'clear')).toBe(false); + await legacy.host.dispose(); + }); + + test('leaves a clear unattributed when the helper does not say how it cleared', async () => { + const { host } = await probedHost(false); + + await expect(host.clear('e1')).resolves.toBeUndefined(); + await host.dispose(); + }); + + test('lets a masked target pause, because a pause delivers nothing', async () => { + const { host, received } = await probedHost(true, ['keyboard', 'physicalClick', 'wheel']); + const pause = [{ actions: [{ duration: 0, type: 'pause' as const }], id: 'keys', type: 'key' as const }]; + + await host.performActions(pause); + await expect(host.performActions([{ actions: [{ type: 'keyDown', value: 'a' }], id: 'keys', type: 'key' }])).rejects.toMatchObject({ + name: 'HostUnsupportedError', + }); + + expect(received.filter((message) => message.command === 'performActions')).toMatchObject([{ params: { actions: pause } }]); + await host.dispose(); + }); +}); diff --git a/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.observation.test.ts b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.observation.test.ts index 525582e6c1..53d853fe73 100644 --- a/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.observation.test.ts +++ b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.observation.test.ts @@ -1,9 +1,7 @@ -import { EventEmitter } from 'node:events'; -import { PassThrough } from 'node:stream'; - import type { NativeObserveRequest } from '../../host/types'; -import type { NativeDriverArtifact, NativeHostJsonMessage } from '../../native/types'; -import { encodeJsonFrame, NativeFrameDecoder } from './framing'; +import type { NativeDriverArtifact } from '../../native/types'; +import { scriptNativeHelper } from '../../testing/scriptedNativeHelper'; +import type { ScriptedNativeRequest, ScriptedNativeSend } from '../../testing/scriptedNativeHelper'; import { NativeDesktopHost } from './NativeDesktopHost'; const mockSpawn = jest.fn(); @@ -42,62 +40,9 @@ const sample = { unsupportedFacts: [], }; -class FakeChild extends EventEmitter { - readonly stdin = new PassThrough(); - readonly stdout = new PassThrough(); - readonly stderr = new PassThrough(); - exitCode: number | null = null; - signalCode: NodeJS.Signals | null = null; - - kill(): boolean { - this.finish(); - return true; - } - - finish(): void { - this.exitCode = 0; - setImmediate(() => { - this.emit('exit', 0, null); - this.emit('close', 0, null); - }); - } -} - -type Send = (message: NativeHostJsonMessage) => void; - /** A host whose helper answers each request through `handler`; the helper acknowledges `dispose` and exits. */ -function createHost(handler: (message: Extract, send: Send) => void) { - const child = new FakeChild(); - const decoder = new NativeFrameDecoder(); - const received: Extract[] = []; - const send: Send = (message) => void child.stdout.write(encodeJsonFrame(message)); - child.stdin.on('data', (chunk: Buffer) => decoder.write(chunk)); - decoder.on('json', (message) => { - if (message.type !== 'request') { - return; - } - if (message.command === 'dispose') { - send({ id: message.id, result: {}, type: 'response' }); - child.finish(); - return; - } - received.push(message); - handler(message, send); - }); - mockSpawn.mockImplementationOnce(() => { - setImmediate(() => - send({ - architecture: 'arm64', - buildId: 'build', - features: ['boundedObservation'], - protocol: { major: 1, minor: 0 }, - provider: 'macos', - sourceDigest: 'source', - type: 'hello', - }), - ); - return child; - }); +function createHost(handler: (message: ScriptedNativeRequest, send: ScriptedNativeSend) => void) { + const received = scriptNativeHelper(mockSpawn, handler, ['boundedObservation']); return { host: new NativeDesktopHost({ application: { windowTitle: 'Test App' }, artifact, endpoint: 'macos' }), received }; } diff --git a/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.ts b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.ts index 4c33f0aeb4..81410b560d 100644 --- a/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.ts +++ b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.ts @@ -2,6 +2,7 @@ import { spawn } from 'node:child_process'; import type { ApplicationLease, + DesktopClearMechanism, DesktopHost, DesktopHostEvent, DesktopHostFeatures, @@ -59,6 +60,8 @@ export class NativeDesktopHost implements DesktopHost { private potentialKeys = new Set(); private failure?: Error; private processPromise?: Promise; + /** What the helper last reported it supports. Every session is created after a probe. */ + private probedFeatures?: DesktopHostFeatures; constructor({ application, artifact, disabledInputFeatures, endpoint, onStderr, onFailure }: NativeDesktopHostOptions) { if (!artifact.endpoints.includes(endpoint)) { @@ -74,7 +77,9 @@ export class NativeDesktopHost implements DesktopHost { async probe(signal?: AbortSignal): Promise { const hostInfo = await this.request('probe', { endpoint: this.endpoint }, signal); - return applyDisabledInputFeaturePolicy(hostInfo, this.disabledInputFeatures); + const masked = applyDisabledInputFeaturePolicy(hostInfo, this.disabledInputFeatures); + this.probedFeatures = masked.features; + return masked; } async launch(target: DesktopTarget, signal?: AbortSignal): Promise { @@ -146,11 +151,27 @@ export class NativeDesktopHost implements DesktopHost { } } - async clear(elementId: string, signal?: AbortSignal): Promise { - if (this.disabledInputFeatures.has('keyboard')) { - throw rejectedBeforeDispatch(new HostUnsupportedError('Physical keyboard fallback is disabled by the registered target policy.')); + /** + * A target that masks the keyboard may still clear semantically, so an unspecified clear is narrowed to the semantic + * mechanism there rather than refused, when the probed helper clears strictly. Without a mechanism a helper that + * predates strict clear reports none, and the clear stays unattributed. + */ + async clear(elementId: string, signal?: AbortSignal, mechanism?: DesktopClearMechanism): Promise { + if (mechanism === 'keyboard' && this.disabledInputFeatures.has('keyboard')) { + throw rejectedBeforeDispatch(new HostUnsupportedError('Physical keyboard input is disabled by the registered target policy.')); + } + const requested = mechanism ?? (this.disabledInputFeatures.has('keyboard') ? 'semantic' : undefined); + if (requested && this.probedFeatures?.strictClear !== true) { + throw rejectedBeforeDispatch( + new HostUnsupportedError( + mechanism + ? `The native helper cannot clear with only the ${mechanism} mechanism.` + : 'Physical keyboard fallback is disabled by the registered target policy, and the native helper cannot clear only semantically.', + ), + ); } - await this.request('clear', { elementId }, signal); + const result = await this.request('clear', { elementId, ...(requested ? { mechanism: requested } : {}) }, signal); + return parseClearMechanism(result); } async sendKeys(elementId: string, text: string, signal?: AbortSignal): Promise { @@ -167,6 +188,8 @@ export class NativeDesktopHost implements DesktopHost { async performActions(actions: readonly NativeActionSequence[], signal?: AbortSignal): Promise { for (const sequence of actions) { + // A source that only pauses delivers nothing, so a masked target may still wait with one. + if (sequence.actions.every((action) => action.type === 'pause')) continue; if ( (sequence.type === 'key' && this.disabledInputFeatures.has('keyboard')) || (sequence.type === 'pointer' && this.disabledInputFeatures.has('physicalClick')) || @@ -547,6 +570,12 @@ const nativeWebDriverErrorCodes: Readonly> = }; /** Marks a failure raised before any command reached the provider, so no effect can have been dispatched. */ +/** The mechanism a helper reports it cleared with. A helper that predates strict clear reports none. */ +function parseClearMechanism(result: unknown): DesktopClearMechanism | undefined { + const mechanism = typeof result === 'object' && result !== null ? (result as { mechanism?: unknown }).mechanism : undefined; + return mechanism === 'keyboard' || mechanism === 'semantic' ? mechanism : undefined; +} + function rejectedBeforeDispatch(error: T): T { return error instanceof Error ? attachFailureEffect(error, 'none') : error; } diff --git a/packages/agentic/desktop-driver/src/index.ts b/packages/agentic/desktop-driver/src/index.ts index 0479787d78..7dc58ae333 100644 --- a/packages/agentic/desktop-driver/src/index.ts +++ b/packages/agentic/desktop-driver/src/index.ts @@ -34,27 +34,40 @@ export { } from './authoring/storyTests.js'; export { accessibleLaneMaskedFeatures, + compareDesktopInputReceipts, + desktopInputMechanisms, + desktopInputModeAllows, + desktopInputModes, desktopInputLanes, desktopRequirements, + describeUndeclaredDesktopInput, matchesDesktopPlatforms, matchesDesktopScope, mergeDesktopCaseOptions, resolveDesktopInput, resolveDesktopScopeDomains, resolveDesktopScopeValues, + subtractDesktopInputReceipts, unmaskedAccessibleFeatures, validateDesktopCaseOptions, validateDesktopGroupOptions, + validateDesktopInputReceipts, + validateDesktopSessionInput, } from './authoring/input.js'; export type { AccessibleLaneMaskedFeature, DesktopCaseOptions, DesktopInput, + DesktopInputComparison, DesktopInputLane, + DesktopInputMechanism, + DesktopInputMode, + DesktopInputReceipts, DesktopPlatforms, DesktopRequirement, DesktopScopeDomains, DesktopScopeValues, + DesktopSessionInput, ResolvedDesktopCaseOptions, } from './authoring/input.js'; export { decideDesktopCase, missingDesktopRequirements, resolveDesktopClickMode } from './authoring/selection.js'; diff --git a/packages/agentic/desktop-driver/src/operations/act.ts b/packages/agentic/desktop-driver/src/operations/act.ts index 0214525a58..a5eccc8a57 100644 --- a/packages/agentic/desktop-driver/src/operations/act.ts +++ b/packages/agentic/desktop-driver/src/operations/act.ts @@ -27,8 +27,8 @@ export type ActionName = (typeof actionNames)[number]; /** * How the provider carried the action out. `physical-pointer` is a synthesized click at a hit-tested point, * `accessibility` is an accessibility action (the element's default one for `activate`, the named one for `perform`), - * `keyboard` is synthesized key input to the focused element, and `provider` means the provider chose and did not say, - * as `clear` does when it can set a value directly. + * `keyboard` is synthesized key input to the focused element, and `provider` means the provider chose and did not say. + * A `clear` reports `accessibility` when it set the value directly and `keyboard` when it typed, when the provider says. */ export const actionMechanisms = ['physical-pointer', 'accessibility', 'keyboard', 'provider'] as const; export type ActionMechanism = (typeof actionMechanisms)[number]; @@ -333,25 +333,34 @@ async function performNamedAction(runtime: OperationRuntime, entry: ReferenceEnt ); } -/** Dispatches one action on a revalidated element. Runs inside the physical input turn. */ +/** + * Dispatches one action on a revalidated element. Runs inside the physical input turn. Resolves with the mechanism when + * the provider decided it, as it does for `clear`. + */ export async function dispatchAction( runtime: OperationRuntime, options: { action: ActionName; name?: string; text?: string }, entry: ReferenceEntry, snapshot: NativeElementSnapshot, -): Promise { +): Promise { const { host } = runtime; switch (options.action) { case 'click': - return physicalClick(runtime, entry, snapshot); + await physicalClick(runtime, entry, snapshot); + return undefined; case 'activate': - return runtime.mutate(`Activating reference ${entry.reference}`, (signal) => host.click(entry.nativeId, 'accessibility', signal)); + await runtime.mutate(`Activating reference ${entry.reference}`, (signal) => host.click(entry.nativeId, 'accessibility', signal)); + return undefined; case 'perform': - return performNamedAction(runtime, entry, options.name!); + await performNamedAction(runtime, entry, options.name!); + return undefined; case 'type': - return runtime.mutate(`Typing into reference ${entry.reference}`, (signal) => host.sendKeys(entry.nativeId, options.text!, signal)); - case 'clear': - return runtime.mutate(`Clearing reference ${entry.reference}`, (signal) => host.clear(entry.nativeId, signal)); + await runtime.mutate(`Typing into reference ${entry.reference}`, (signal) => host.sendKeys(entry.nativeId, options.text!, signal)); + return undefined; + case 'clear': { + const used = await runtime.mutate(`Clearing reference ${entry.reference}`, (signal) => host.clear(entry.nativeId, signal)); + return used === 'semantic' ? 'accessibility' : used; + } } } @@ -363,6 +372,7 @@ export async function performAction(runtime: OperationRuntime, input: ActOptions assertNotQuarantined(runtime); assertActionSupported(runtime, input.action); const { entry, proof } = input.find ? await proveSelector(runtime, input.find) : resolveProven(runtime, input.ref!, input.sample!); + let delivered: ActionMechanism | undefined; try { await runtime.input(async () => { if (runtime.signal.aborted) { @@ -373,7 +383,7 @@ export async function performAction(runtime: OperationRuntime, input: ActOptions } const snapshot = await revalidate(runtime, entry, proof); requireInteractable(runtime, input.action, input.ref, snapshot); - await dispatchAction(runtime, input, entry, snapshot); + delivered = await dispatchAction(runtime, input, entry, snapshot); }); } catch (error) { // Every primary mutation settles through `runtime.mutate`, which reports what it did; a failure without a report @@ -384,7 +394,7 @@ export async function performAction(runtime: OperationRuntime, input: ActOptions return { action: input.action, disposition: 'applied', - mechanism: mechanismOf[input.action], + mechanism: delivered ?? mechanismOf[input.action], ref: entry.reference, sample: proof.sample, ...(warnings.length > 0 ? { warnings } : {}), diff --git a/packages/agentic/desktop-driver/src/operations/runtime.ts b/packages/agentic/desktop-driver/src/operations/runtime.ts index d8e0baa0db..ce90ac0bbf 100644 --- a/packages/agentic/desktop-driver/src/operations/runtime.ts +++ b/packages/agentic/desktop-driver/src/operations/runtime.ts @@ -16,7 +16,8 @@ export type OperationHost = { activate(windowId: string, signal: AbortSignal): Promise; captureElement(nativeId: string, signal: AbortSignal): Promise; captureWindow(windowId: string, signal: AbortSignal): Promise; - clear(nativeId: string, signal: AbortSignal): Promise; + /** Resolves with how the value was cleared, or nothing when the provider did not say. */ + clear(nativeId: string, signal: AbortSignal): Promise<'keyboard' | 'semantic' | undefined>; click(nativeId: string, mode: 'accessibility' | 'physical', signal: AbortSignal): Promise; hitTest(windowId: string, x: number, y: number, signal: AbortSignal): Promise; /** The element's published actions with their keys. Called only when `capabilities.accessibilityActions` is true. */ diff --git a/packages/agentic/desktop-driver/src/protocol/capabilities.ts b/packages/agentic/desktop-driver/src/protocol/capabilities.ts index 5e8eee1716..12c2ac4141 100644 --- a/packages/agentic/desktop-driver/src/protocol/capabilities.ts +++ b/packages/agentic/desktop-driver/src/protocol/capabilities.ts @@ -1,3 +1,5 @@ +import { desktopInputModes } from '../authoring/input.js'; +import type { DesktopInputMode } from '../authoring/input.js'; import type { DesktopHostInfo, DesktopTarget } from '../host/types.js'; import { invalidArgument, toWebDriverError, WebDriverError } from './errors.js'; import { withCommandTimeout } from './timeouts.js'; @@ -20,6 +22,7 @@ const standardCapabilities = new Set([ export type MatchedCapabilities = { clickMode: DesktopClickMode; hostInfo: DesktopHostInfo; + inputMode: DesktopInputMode; launchMode: 'attach' | 'launch'; requested: Record; target: DesktopTarget; @@ -54,6 +57,7 @@ export async function matchCapabilities( for (const requested of candidates) { validateCapabilityNames(requested); const requestedClickMode = validateClickMode(requested['furn:clickMode']); + const inputMode = validateInputMode(requested['furn:inputMode']); const launchMode = validateLaunchMode(requested['furn:launchMode']); const targetId = requested['furn:target']; const candidatesForTarget = @@ -80,8 +84,8 @@ export async function matchCapabilities( const webdriverError = toWebDriverError(error); throw new WebDriverError('session not created', webdriverError.message, webdriverError.data); } - const clickMode = resolveClickMode(requestedClickMode, host); - return { clickMode, hostInfo: host, launchMode, requested, target }; + const clickMode = resolveClickMode(requestedClickMode, host, inputMode); + return { clickMode, hostInfo: host, inputMode, launchMode, requested, target }; } } @@ -101,6 +105,7 @@ export function createReturnedCapabilities( 'furn:clickMode': matched.clickMode, 'furn:endpoint': matched.target.endpoint, 'furn:features': host.features, + 'furn:inputMode': matched.inputMode, ...(matched.target.renderer ? { 'furn:renderer': matched.target.renderer } : {}), 'furn:storybook': Boolean(matched.target.storyOrchestrator), 'furn:target': matched.target.id, @@ -130,6 +135,14 @@ function validateClickMode(requested: unknown): DesktopClickMode | 'auto' { return clickMode; } +function validateInputMode(requested: unknown): DesktopInputMode { + const inputMode = requested ?? 'compatibility'; + if (!(desktopInputModes as readonly unknown[]).includes(inputMode)) { + throw invalidArgument(`"furn:inputMode" must be one of: ${desktopInputModes.join(', ')}.`); + } + return inputMode as DesktopInputMode; +} + function validateLaunchMode(requested: unknown): 'attach' | 'launch' { const launchMode = requested ?? 'launch'; if (launchMode !== 'attach' && launchMode !== 'launch') { @@ -138,7 +151,18 @@ function validateLaunchMode(requested: unknown): 'attach' | 'launch' { return launchMode; } -function resolveClickMode(clickMode: DesktopClickMode | 'auto', host: DesktopHostInfo): DesktopClickMode { +/** + * A strict input mode decides the click transport, so a conflicting request fails. It does not require the host to + * click at all, because a session may only type or clear; a click the host cannot deliver is refused when it is sent. + */ +function resolveClickMode(clickMode: DesktopClickMode | 'auto', host: DesktopHostInfo, inputMode: DesktopInputMode): DesktopClickMode { + if (inputMode !== 'compatibility') { + const pinned: DesktopClickMode = inputMode === 'accessible' ? 'accessibility' : 'physical'; + if (clickMode !== 'auto' && clickMode !== pinned) { + throw new WebDriverError('session not created', `"furn:clickMode" "${clickMode}" conflicts with "furn:inputMode" "${inputMode}".`); + } + return pinned; + } if (clickMode === 'physical' && !host.features.physicalClick) { throw new WebDriverError('session not created', 'The target does not support physical click input.'); } diff --git a/packages/agentic/desktop-driver/src/server/SessionManager.ts b/packages/agentic/desktop-driver/src/server/SessionManager.ts index 8b430c3741..5df7b5ba60 100644 --- a/packages/agentic/desktop-driver/src/server/SessionManager.ts +++ b/packages/agentic/desktop-driver/src/server/SessionManager.ts @@ -1,5 +1,6 @@ import { randomUUID } from 'node:crypto'; +import type { DesktopInputMechanism, DesktopInputMode } from '../authoring/input.js'; import type { ApplicationLease, DesktopHostInfo, @@ -97,6 +98,10 @@ export type DesktopSession = { elementIdsByNativeId: Map; hostInfo: DesktopHostInfo; id: string; + /** The input policy fixed at creation (`furn:inputMode`). It never changes for the life of the session. */ + readonly inputMode: DesktopInputMode; + /** How many times the session delivered input by each mechanism, including deliveries whose outcome was not observed. */ + inputReceipts: Map; inputState: WebDriverInputState; issuedElementIds: Set; lease: ApplicationLease; @@ -249,6 +254,7 @@ export class SessionManager { clickMode: DesktopClickMode, launchMode: 'attach' | 'launch', owner?: SessionOwner, + inputMode: DesktopInputMode = 'compatibility', ): Promise { if (this.closing) { return Promise.reject(new WebDriverError('session not created', 'Desktop Driver is shutting down.')); @@ -259,7 +265,7 @@ export class SessionManager { return Promise.reject(new WebDriverError('session not created', 'The grant for this request was revoked.')); } const controller = new AbortController(); - const result = this.createSession(target, hostInfo, clickMode, launchMode, owner, controller.signal); + const result = this.createSession(target, hostInfo, clickMode, launchMode, inputMode, owner, controller.signal); const settled = result.then( () => undefined, () => undefined, @@ -300,6 +306,7 @@ export class SessionManager { hostInfo: DesktopHostInfo, clickMode: DesktopClickMode, launchMode: 'attach' | 'launch', + inputMode: DesktopInputMode, owner?: SessionOwner, signal?: AbortSignal, ): Promise { @@ -375,6 +382,8 @@ export class SessionManager { elementIdsByNativeId: new Map(), hostInfo, id: randomUUID(), + inputMode, + inputReceipts: new Map(), inputState: createInputState(), issuedElementIds: new Set(), lease, diff --git a/packages/agentic/desktop-driver/src/server/createDesktopDriverServer.ts b/packages/agentic/desktop-driver/src/server/createDesktopDriverServer.ts index b2dac9e849..f3b7c20f48 100644 --- a/packages/agentic/desktop-driver/src/server/createDesktopDriverServer.ts +++ b/packages/agentic/desktop-driver/src/server/createDesktopDriverServer.ts @@ -2,7 +2,15 @@ import { Buffer } from 'node:buffer'; import { createServer } from 'node:http'; import type { IncomingMessage, Server, ServerResponse } from 'node:http'; -import type { DesktopTarget, DesktopTreeNode, NativeActionSequence, NativeElementSnapshot, NativeSelector, Rect } from '../host/types.js'; +import type { + DesktopTarget, + DesktopTreeNode, + NativeAccessibilityAction, + NativeActionSequence, + NativeElementSnapshot, + NativeSelector, + Rect, +} from '../host/types.js'; import { DesktopAuthority, grantAllowsTarget } from '../authority/grants.js'; import type { DesktopGrant } from '../authority/grants.js'; import type { PrivateStorage } from '../authority/privateStorage.js'; @@ -15,6 +23,15 @@ import { withCommandTimeout } from '../protocol/timeouts.js'; import type { NewSessionRequest, WebDriverActionSequence, WebDriverElement, WebDriverResponse } from '../protocol/types.js'; import { SessionManager } from './SessionManager.js'; import type { DesktopSession, ElementRecord, MutationOptions, RecoverTargetRequest, RecoverTargetResult } from './SessionManager.js'; +import { + actionMechanisms, + assertInputAllowed, + clickMechanism, + deliverClear, + deliverInput, + describeSessionInput, + strictClearMechanism, +} from './sessionInput.js'; import { SettlementJournal } from './settlementJournal.js'; import type { ProbeApplication } from './settlementJournal.js'; import { createOperationRuntime, handleOperationRoute, requestBodyLimit } from './sessionOperations.js'; @@ -342,7 +359,14 @@ async function route(context: RouteContext, sessions: SessionManager, targets: T const matched = await matchCapabilities(request.capabilities, visibleTargets(context, targets)); try { const owner = context.grant ? { grantId: context.grant.id, label: context.grant.label } : undefined; - const session = await sessions.create(matched.target, matched.hostInfo, matched.clickMode, matched.launchMode, owner); + const session = await sessions.create( + matched.target, + matched.hostInfo, + matched.clickMode, + matched.launchMode, + owner, + matched.inputMode, + ); return { sessionId: session.id, capabilities: createReturnedCapabilities(matched, matched.hostInfo, session.timeouts), @@ -390,12 +414,16 @@ async function route(context: RouteContext, sessions: SessionManager, targets: T if (method === 'POST') { const { actions, nextState } = parseActionSequences(requireObject(body, 'actions request').actions, session.inputState); const resolvedActions = await resolveActionOrigins(actions, session, sessions); + const mechanisms = actionMechanisms(resolvedActions); + for (const mechanism of mechanisms) { + assertInputAllowed(session, mechanism, 'Performing input actions'); + } await sessions.runInputCommand(async () => { sessions.assertMutable(session); session.inputState = nextState; try { await mutationCommand(sessions, session, 'Performing input actions', (signal) => - session.target.host.performActions(resolvedActions, signal), + deliverInput(session, mechanisms, () => session.target.host.performActions(resolvedActions, signal)), ); } catch (error) { await mutationCommand( @@ -433,6 +461,9 @@ async function route(context: RouteContext, sessions: SessionManager, targets: T ); } if (command[0] === 'furn') { + if (method === 'GET' && command[1] === 'input' && command.length === 2) { + return describeSessionInput(session); + } if (method === 'GET' && command[1] === 'tree' && command.length === 2) { const snapshots = await hostCommand(session, 'Reading the compact accessibility tree', (signal) => session.target.host.tree(session.currentWindowId, signal), @@ -729,6 +760,9 @@ async function handleElementCommand( if (!requireSupported(snapshot.enabled, 'enabled') || !requireSupported(snapshot.visible, 'visible')) { throw new WebDriverError('element not interactable', `Element "${record.id}" cannot be clicked.`); } + const mechanism = clickMechanism(session.clickMode); + // Checked before the window is activated, so a refused click changes nothing. + assertInputAllowed(session, mechanism, `Clicking element "${record.id}"`); await sessions.runInputCommand(async () => { if (session.clickMode !== 'accessibility') { await mutationCommand(sessions, session, `Activating window "${snapshot.windowId}"`, (signal) => @@ -749,14 +783,18 @@ async function handleElementCommand( } } await mutationCommand(sessions, session, `Clicking element "${record.id}"`, (signal) => - session.target.host.click(record.nativeId, session.clickMode, signal), + deliverInput(session, [mechanism], () => session.target.host.click(record.nativeId, session.clickMode, signal)), ); }); return null; } if (method === 'POST' && operation[0] === 'clear') { + const description = `Clearing element "${record.id}"`; + const requested = strictClearMechanism(session, description); await sessions.runInputCommand(() => - mutationCommand(sessions, session, `Clearing element "${record.id}"`, (signal) => session.target.host.clear(record.nativeId, signal)), + mutationCommand(sessions, session, description, (signal) => + deliverClear(session, requested, description, (mechanism) => session.target.host.clear(record.nativeId, signal, mechanism)), + ), ); return null; } @@ -771,13 +809,17 @@ async function handleElementCommand( if (text === undefined) { throw invalidArgument('Send Keys requires a string "text" or string-array "value".'); } + assertInputAllowed(session, 'keyboard', `Sending keys to element "${record.id}"`); await sessions.runInputCommand(() => mutationCommand(sessions, session, `Sending keys to element "${record.id}"`, (signal) => - session.target.host.sendKeys(record.nativeId, text, signal), + deliverInput(session, ['keyboard'], () => session.target.host.sendKeys(record.nativeId, text, signal)), ), ); return null; } + if (method === 'POST' && operation[0] === 'furn' && operation[1] === 'accessibility-action' && operation.length === 2) { + return performNamedAccessibilityAction(body, record, session, sessions); + } if (method === 'GET' && operation[0] === 'screenshot') { const image = await hostCommand(session, `Capturing element "${record.id}"`, (signal) => session.target.host.captureElement(record.nativeId, signal), @@ -788,6 +830,53 @@ async function handleElementCommand( throw new WebDriverError('unknown command', `Unknown element command "${operation.join('/')}".`); } +/** + * Performs the one accessibility action an element publishes under a name, such as a custom `Toggle`. The action is + * listed and performed in the same input turn, and a name the element publishes twice names neither action. + */ +async function performNamedAccessibilityAction( + body: unknown, + record: ElementRecord, + session: DesktopSession, + sessions: SessionManager, +): Promise { + const request = requireObject(body, 'accessibility action request'); + if (typeof request.name !== 'string' || request.name.length === 0) { + throw invalidArgument('An accessibility action request requires a non-empty string "name".'); + } + const name = request.name; + const host = session.target.host; + const list = host.listAccessibilityActions?.bind(host); + const perform = host.performAccessibilityAction?.bind(host); + if (!list || !perform || session.hostInfo.features.performAccessibilityAction !== true) { + throw new WebDriverError('unsupported operation', `Target "${session.target.id}" does not perform accessibility actions.`); + } + const description = `Performing an accessibility action on element "${record.id}"`; + assertInputAllowed(session, 'accessibility-action', description); + await sessions.runInputCommand(async () => { + const actions: NativeAccessibilityAction[] = await hostCommand( + session, + `Listing the accessibility actions of element "${record.id}"`, + (signal) => list(record.nativeId, signal), + ); + const matches = actions.filter((action) => action.name === name); + if (matches.length !== 1) { + throw new WebDriverError( + 'element not interactable', + matches.length === 0 + ? `Element "${record.id}" publishes no accessibility action by that name.` + : `Element "${record.id}" publishes ${matches.length} accessibility actions by that name, so the name names none of them.`, + { reason: matches.length === 0 ? 'action-not-published' : 'action-not-unique' }, + ); + } + const [{ key }] = matches; + await mutationCommand(sessions, session, description, (signal) => + deliverInput(session, ['accessibility-action'], () => perform(record.nativeId, key, signal)), + ); + }); + return null; +} + async function isElementOrDescendant(session: DesktopSession, targetNativeId: string, hit: NativeElementSnapshot): Promise { let current: NativeElementSnapshot | undefined = hit; for (let depth = 0; current && depth < 100; depth += 1) { diff --git a/packages/agentic/desktop-driver/src/server/desktopDriver.test.ts b/packages/agentic/desktop-driver/src/server/desktopDriver.test.ts index 66e56de0f4..24d7c35068 100644 --- a/packages/agentic/desktop-driver/src/server/desktopDriver.test.ts +++ b/packages/agentic/desktop-driver/src/server/desktopDriver.test.ts @@ -245,6 +245,7 @@ describe('Desktop Driver W3C remote end', () => { test.each([ ['furn:clickMode', 'not-a-click-mode'], + ['furn:inputMode', 'strict'], ['furn:launchMode', 'not-a-launch-mode'], ])('rejects invalid %s before probing the target', async (capability, value) => { const harness = await createDesktopDriverTestHarness(); diff --git a/packages/agentic/desktop-driver/src/server/operations.test.ts b/packages/agentic/desktop-driver/src/server/operations.test.ts index 5a514e5381..e308562b87 100644 --- a/packages/agentic/desktop-driver/src/server/operations.test.ts +++ b/packages/agentic/desktop-driver/src/server/operations.test.ts @@ -505,7 +505,7 @@ describe('Desktop Driver agent operations', () => { await expect(session.act({ action: 'type', text: 'secret words', ...field })).resolves.toMatchObject({ result: { mechanism: 'keyboard' }, }); - await expect(session.act({ action: 'clear', ...field })).resolves.toMatchObject({ result: { mechanism: 'provider' } }); + await expect(session.act({ action: 'clear', ...field })).resolves.toMatchObject({ result: { mechanism: 'accessibility' } }); const dispatched = harness.host.actions.map(({ elementId, ...action }) => // The fake issues a fresh record id, such as `save~2`, each time an element is observed again. @@ -517,7 +517,7 @@ describe('Desktop Driver agent operations', () => { { elementId: 'agree', mode: 'physical', type: 'click' }, { elementId: 'save', mode: 'accessibility', type: 'click' }, { elementId: 'field', text: 'secret words', type: 'send-keys' }, - { elementId: 'field', type: 'clear' }, + { elementId: 'field', mechanism: 'semantic', type: 'clear' }, ]), ); const journal = await session.journal(); @@ -525,7 +525,7 @@ describe('Desktop Driver agent operations', () => { { effect: 'mutate', outcome: 'ok', summary: { action: 'click', mechanism: 'physical-pointer' } }, { summary: { action: 'activate', mechanism: 'accessibility' } }, { summary: { action: 'type', mechanism: 'keyboard' } }, - { summary: { action: 'clear', mechanism: 'provider' } }, + { summary: { action: 'clear', mechanism: 'accessibility' } }, ]); expect(JSON.stringify(journal.result)).not.toContain('secret words'); }); diff --git a/packages/agentic/desktop-driver/src/server/sessionInput.test.ts b/packages/agentic/desktop-driver/src/server/sessionInput.test.ts new file mode 100644 index 0000000000..e61a024cb8 --- /dev/null +++ b/packages/agentic/desktop-driver/src/server/sessionInput.test.ts @@ -0,0 +1,272 @@ +import { createDesktopDriverClient } from '../client/DesktopDriverClient.js'; +import type { DesktopSessionClient } from '../client/DesktopDriverClient.js'; +import { compareDesktopInputReceipts, validateDesktopCaseOptions } from '../authoring/input.js'; +import type { DesktopInputMode } from '../authoring/input.js'; +import type { FakeDesktopHostOptions } from '../hosts/fake/FakeDesktopHost.js'; +import type { WebDriverActionSequence } from '../protocol/types.js'; +import { createDesktopDriverTestHarness } from '../testing/protocolHarness.js'; +import type { DesktopDriverTestHarness } from '../testing/protocolHarness.js'; + +type Fixture = { harness: DesktopDriverTestHarness; session: DesktopSessionClient }; + +const element = (id: string, y: number, extra: Record = {}) => ({ + automationId: id, + id, + name: id, + parentId: 'root', + rect: { height: 30, width: 100, x: 10, y }, + role: 'button', + scope: 'application' as const, + windowId: 'main', + ...extra, +}); + +const windows: FakeDesktopHostOptions['windows'] = [ + { + elements: [ + { id: 'root', name: 'Form', rect: { height: 600, width: 800, x: 0, y: 0 }, role: 'window', scope: 'application', windowId: 'main' }, + element('save', 10), + element('field', 50, { role: 'textbox', value: 'draft' }), + element('label', 90, { role: 'text' }), + element('stepper', 130, { accessibilityActions: ['Increment', 'Decrement', 'Reset', 'Reset'] }), + ], + id: 'main', + title: 'Input fixture', + }, +]; + +const allFeatures = { accessibilityActions: true, boundedObservation: true, performAccessibilityAction: true }; + +async function withSession( + inputMode: DesktopInputMode | undefined, + body: (fixture: Fixture) => Promise, + options: FakeDesktopHostOptions = {}, +): Promise { + const harness = await createDesktopDriverTestHarness({ windows, ...options, features: { ...allFeatures, ...options.features } }); + try { + const session = await createDesktopDriverClient({ url: harness.server.url }).newSession({ + alwaysMatch: { platformName: 'windows', 'furn:target': harness.target.id, ...(inputMode ? { 'furn:inputMode': inputMode } : {}) }, + }); + await body({ harness, session }); + await session.delete(); + } finally { + await harness.close(); + } +} + +const refused = (reason: 'input-policy' | 'input-unsupported', mechanism: string) => ({ + code: 'unsupported operation', + data: expect.objectContaining({ mechanism, reason }), +}); + +const pointerDown: WebDriverActionSequence[] = [{ actions: [{ button: 0, type: 'pointerDown' }], id: 'mouse', type: 'pointer' }]; +const keyDown: WebDriverActionSequence[] = [{ actions: [{ type: 'keyDown', value: 'a' }], id: 'keys', type: 'key' }]; +const scroll: WebDriverActionSequence[] = [ + { actions: [{ deltaX: 0, deltaY: 10, type: 'scroll', x: 0, y: 0 }], id: 'wheel', type: 'wheel' }, +]; + +describe('Desktop Driver session input policy', () => { + test('defaults to compatibility, and reports the mode it enforces', async () => { + await withSession(undefined, async ({ session }) => { + expect(session.capabilities['furn:inputMode']).toBe('compatibility'); + await expect(session.getInput()).resolves.toEqual({ mode: 'compatibility', receipts: {} }); + }); + await withSession('accessible', async ({ session }) => { + expect(session.capabilities).toMatchObject({ 'furn:clickMode': 'accessibility', 'furn:inputMode': 'accessible' }); + }); + await withSession('physical', async ({ session }) => { + expect(session.capabilities).toMatchObject({ 'furn:clickMode': 'physical', 'furn:inputMode': 'physical' }); + }); + }); + + test('refuses a click mode that conflicts with a strict input mode', async () => { + const harness = await createDesktopDriverTestHarness(); + try { + const client = createDesktopDriverClient({ url: harness.server.url }); + for (const [inputMode, clickMode] of [ + ['accessible', 'physical'], + ['physical', 'accessibility'], + ]) { + await expect( + client.newSession({ + alwaysMatch: { + platformName: 'windows', + 'furn:clickMode': clickMode, + 'furn:inputMode': inputMode, + 'furn:target': harness.target.id, + }, + }), + ).rejects.toMatchObject({ code: 'session not created' }); + } + } finally { + await harness.close(); + } + }); + + test('an accessible session delivers only semantic input, and dispatches nothing it refuses', async () => { + await withSession('accessible', async ({ harness, session }) => { + const save = await session.findElement('accessibility id', 'save'); + const field = await session.findElement('accessibility id', 'field'); + await save.click(); + await field.clear(); + await expect(field.sendKeys('secret')).rejects.toMatchObject(refused('input-policy', 'keyboard')); + await expect(session.performActions(pointerDown)).rejects.toMatchObject(refused('input-policy', 'pointer')); + await expect(session.performActions(keyDown)).rejects.toMatchObject(refused('input-policy', 'keyboard')); + await expect(session.performActions(scroll)).rejects.toMatchObject(refused('input-policy', 'wheel')); + // Waiting delivers nothing, so an accessible case may still pause. + await session.performActions([{ actions: [{ duration: 0, type: 'pause' }], id: 'idle', type: 'none' }]); + await session.performActions([{ actions: [{ duration: 0, type: 'pause' }], id: 'keys', type: 'key' }]); + + expect(harness.host.actions.filter((action) => ['actions', 'clear', 'click', 'send-keys'].includes(action.type as string))).toEqual([ + { elementId: 'save', mode: 'accessibility', type: 'click' }, + { elementId: 'field', mechanism: 'semantic', type: 'clear' }, + expect.objectContaining({ actions: [expect.objectContaining({ id: 'idle' })], type: 'actions' }), + expect.objectContaining({ actions: [expect.objectContaining({ id: 'keys' })], type: 'actions' }), + ]); + await expect(session.getInput()).resolves.toEqual({ + mode: 'accessible', + receipts: { 'accessibility-click': 1, 'semantic-clear': 1 }, + }); + }); + }); + + test('an accessible session refuses a clear the target cannot make semantically', async () => { + await withSession('accessible', async ({ harness, session }) => { + await expect((await session.findElement('accessibility id', 'label')).clear()).rejects.toMatchObject({ + code: 'element not interactable', + }); + expect(harness.host.actions.filter((action) => action.type === 'clear')).toEqual([]); + await expect(session.getInput()).resolves.toEqual({ mode: 'accessible', receipts: {} }); + }); + await withSession( + 'accessible', + async ({ harness, session }) => { + await expect((await session.findElement('accessibility id', 'field')).clear()).rejects.toMatchObject( + refused('input-unsupported', 'semantic-clear'), + ); + expect(harness.host.actions.filter((action) => action.type === 'clear')).toEqual([]); + }, + { features: { strictClear: false } }, + ); + }); + + test('a physical session clears only with the keyboard, and needs only the keyboard to start', async () => { + await withSession( + 'physical', + async ({ harness, session }) => { + const field = await session.findElement('accessibility id', 'field'); + await field.clear(); + await field.sendKeys('typed'); + await expect((await session.findElement('accessibility id', 'save')).click()).rejects.toMatchObject( + refused('input-unsupported', 'physical-click'), + ); + expect(harness.host.actions.filter((action) => action.type === 'clear' || action.type === 'click')).toEqual([ + { elementId: 'field', mechanism: 'keyboard', type: 'clear' }, + ]); + await expect(session.getInput()).resolves.toEqual({ mode: 'physical', receipts: { keyboard: 2 } }); + }, + { features: { accessibilityClick: false, physicalClick: false } }, + ); + }); + + test('fails a strict clear that the host made another way, after recording what it delivered', async () => { + await withSession('physical', async ({ harness, session }) => { + Object.assign(harness.host, { clear: async () => 'semantic' }); + await expect((await session.findElement('accessibility id', 'field')).clear()).rejects.toMatchObject({ + code: 'unknown error', + data: expect.objectContaining({ reason: 'input-mechanism-mismatch' }), + }); + await expect(session.getInput()).resolves.toEqual({ mode: 'physical', receipts: { 'semantic-clear': 1 } }); + }); + }); + + test('counts a delivery unless its failure proves nothing was dispatched, and attributes what compatibility clears', async () => { + await withSession(undefined, async ({ harness, session }) => { + const save = await session.findElement('accessibility id', 'save'); + harness.host.injectFault('click', { effect: 'none', mode: 'reject' }); + await expect(save.click()).rejects.toBeDefined(); + harness.host.injectFault('click', { effect: 'applied', mode: 'reject' }); + await expect(save.click()).rejects.toBeDefined(); + await save.click(); + const field = await session.findElement('accessibility id', 'field'); + await field.clear(); + Object.assign(harness.host, { clear: async () => undefined }); + await field.clear(); + await expect(session.getInput()).resolves.toEqual({ + mode: 'compatibility', + receipts: { 'physical-click': 2, 'semantic-clear': 1, 'unattributed-clear': 1 }, + }); + }); + }); + + test('performs exactly one published accessibility action by name', async () => { + await withSession('accessible', async ({ harness, session }) => { + const stepper = await session.findElement('accessibility id', 'stepper'); + await stepper.performAccessibilityAction('Increment'); + await expect(stepper.performAccessibilityAction('Toggle')).rejects.toMatchObject({ + code: 'element not interactable', + data: expect.objectContaining({ reason: 'action-not-published' }), + }); + await expect(stepper.performAccessibilityAction('Reset')).rejects.toMatchObject({ + code: 'element not interactable', + data: expect.objectContaining({ reason: 'action-not-unique' }), + }); + await expect(stepper.performAccessibilityAction('')).rejects.toMatchObject({ code: 'invalid argument' }); + expect(harness.host.actions.filter((action) => action.type === 'accessibility-action')).toEqual([ + { elementId: 'stepper', name: 'Increment', type: 'accessibility-action' }, + ]); + await expect(session.getInput()).resolves.toEqual({ mode: 'accessible', receipts: { 'accessibility-action': 1 } }); + }); + await withSession( + 'accessible', + async ({ session }) => { + await expect( + (await session.findElement('accessibility id', 'stepper')).performAccessibilityAction('Increment'), + ).rejects.toMatchObject({ + code: 'unsupported operation', + }); + }, + { features: { performAccessibilityAction: false } }, + ); + }); + + test('a physical session also delivers the accessible transports a mixed case declares, and receipts account for each', async () => { + await withSession('physical', async ({ session }) => { + await (await session.findElement('accessibility id', 'field')).sendKeys('typed'); + await (await session.findElement('accessibility id', 'stepper')).performAccessibilityAction('Increment'); + const { receipts } = await session.getInput(); + + expect(receipts).toEqual({ 'accessibility-action': 1, keyboard: 1 }); + const mixed = validateDesktopCaseOptions({ input: 'physical', requires: ['keyboard', 'perform-accessibility-action'] }); + expect(compareDesktopInputReceipts(mixed, receipts, 'physical')).toEqual({ undeclared: [], unused: [] }); + expect(compareDesktopInputReceipts(validateDesktopCaseOptions({ requires: ['keyboard'] }), receipts, 'physical').undeclared).toEqual([ + 'accessibility-action', + ]); + }); + }); + + test('agent operations in an accessible session never type or move the pointer', async () => { + await withSession('accessible', async ({ harness, session }) => { + await expect(session.act({ action: 'click', find: { automationId: 'save' } })).rejects.toMatchObject({ + code: 'unsupported operation', + }); + await expect(session.act({ action: 'type', find: { automationId: 'field' }, text: 'secret' })).rejects.toMatchObject({ + code: 'unsupported operation', + }); + await expect(session.act({ action: 'activate', find: { automationId: 'save' } })).resolves.toMatchObject({ + result: { mechanism: 'accessibility' }, + }); + await expect(session.act({ action: 'clear', find: { automationId: 'field' } })).resolves.toMatchObject({ + result: { mechanism: 'accessibility' }, + }); + await expect(session.act({ action: 'perform', find: { automationId: 'stepper' }, name: 'Increment' })).resolves.toBeDefined(); + expect( + harness.host.actions.some((action) => action.type === 'send-keys' || (action.type === 'click' && action.mode === 'physical')), + ).toBe(false); + await expect(session.getInput()).resolves.toEqual({ + mode: 'accessible', + receipts: { 'accessibility-action': 1, 'accessibility-click': 1, 'semantic-clear': 1 }, + }); + }); + }); +}); diff --git a/packages/agentic/desktop-driver/src/server/sessionInput.ts b/packages/agentic/desktop-driver/src/server/sessionInput.ts new file mode 100644 index 0000000000..1a606c4e64 --- /dev/null +++ b/packages/agentic/desktop-driver/src/server/sessionInput.ts @@ -0,0 +1,155 @@ +import { desktopInputMechanisms, desktopInputModeAllows } from '../authoring/input.js'; +import type { DesktopInputMechanism, DesktopSessionInput } from '../authoring/input.js'; +import type { DesktopClearMechanism, DesktopHostFeatures, NativeActionSequence } from '../host/types.js'; +import { WebDriverError } from '../protocol/errors.js'; +import { attachFailureEffect, readFailureEffect } from '../protocol/settlement.js'; +import type { DesktopClickMode } from '../protocol/types.js'; +import type { DesktopSession } from './SessionManager.js'; + +/** The host feature a strict session needs before it delivers each mechanism. */ +const mechanismFeature: Readonly> = { + 'accessibility-action': 'performAccessibilityAction', + 'accessibility-click': 'accessibilityClick', + keyboard: 'keyboard', + 'physical-click': 'physicalClick', + pointer: 'physicalClick', + 'semantic-clear': 'strictClear', + 'unattributed-clear': undefined, + wheel: 'wheel', +}; + +/** What `GET /session/:id/furn/input` reports. */ +export function describeSessionInput(session: DesktopSession): DesktopSessionInput { + const receipts: Partial> = {}; + for (const mechanism of desktopInputMechanisms) { + const count = session.inputReceipts.get(mechanism); + if (count) receipts[mechanism] = count; + } + return { mode: session.inputMode, receipts }; +} + +export function clickMechanism(mode: DesktopClickMode): DesktopInputMechanism { + return mode === 'accessibility' ? 'accessibility-click' : 'physical-click'; +} + +/** + * The mechanisms a sequence of actions delivers. A source that only pauses delivers nothing, so a strict session may + * still wait with one; a pointer that moves without pressing still moves the real pointer. + */ +export function actionMechanisms(sequences: readonly NativeActionSequence[]): DesktopInputMechanism[] { + const mechanisms = new Set(); + for (const sequence of sequences) { + if (!sequence.actions.some((action) => action.type !== 'pause')) continue; + if (sequence.type === 'key') mechanisms.add('keyboard'); + else if (sequence.type === 'pointer') mechanisms.add('pointer'); + else if (sequence.type === 'wheel') mechanisms.add('wheel'); + } + return [...mechanisms]; +} + +/** + * Refuses input a strict session's policy or host cannot deliver, before anything is dispatched, so the refusal is a + * settled rejection. A compatibility session leaves the decision to the host, as it always has. + */ +export function assertInputAllowed(session: DesktopSession, mechanism: DesktopInputMechanism, description: string): void { + const mode = session.inputMode; + if (mode === 'compatibility') return; + if (!desktopInputModeAllows(mode, mechanism)) { + throw refuse(`${description} would deliver ${mechanism} input, which a session with input mode "${mode}" never delivers.`, { + inputMode: mode, + mechanism, + reason: 'input-policy', + }); + } + const feature = mechanismFeature[mechanism]; + if (feature && session.hostInfo.features[feature] !== true) { + throw refuse(`${description} would deliver ${mechanism} input, which target "${session.target.id}" does not deliver.`, { + inputMode: mode, + mechanism, + reason: 'input-unsupported', + }); + } +} + +/** The clear mechanism a strict session requires, after checking the host can clear with only that mechanism. */ +export function strictClearMechanism(session: DesktopSession, description: string): DesktopClearMechanism | undefined { + const mode = session.inputMode; + if (mode === 'compatibility') return undefined; + const mechanism: DesktopClearMechanism = mode === 'accessible' ? 'semantic' : 'keyboard'; + assertInputAllowed(session, mechanism === 'semantic' ? 'semantic-clear' : 'keyboard', description); + if (session.hostInfo.features.strictClear !== true) { + throw refuse(`${description} needs a ${mechanism} clear, but target "${session.target.id}" cannot clear with only one mechanism.`, { + inputMode: mode, + mechanism: mechanism === 'semantic' ? 'semantic-clear' : 'keyboard', + reason: 'input-unsupported', + }); + } + return mechanism; +} + +/** + * Dispatches one delivery of input and records its receipt. A failure counts unless it proves nothing was dispatched, + * because a delivery whose outcome was not observed may still have reached the application. + */ +export async function deliverInput( + session: DesktopSession, + mechanisms: readonly DesktopInputMechanism[], + dispatch: () => Promise, +): Promise { + try { + const result = await dispatch(); + recordInput(session, mechanisms); + return result; + } catch (error) { + if (readFailureEffect(error) !== 'none') recordInput(session, mechanisms); + throw error; + } +} + +/** + * Clears and records how. A strict session asks for one mechanism and fails a host that used another or would not say, + * after recording what was delivered. A compatibility session records what the host reports, or an unattributed clear. + */ +export async function deliverClear( + session: DesktopSession, + requested: DesktopClearMechanism | undefined, + description: string, + dispatch: (mechanism: DesktopClearMechanism | undefined) => Promise, +): Promise { + let used: DesktopClearMechanism | void; + try { + used = await dispatch(requested); + } catch (error) { + if (readFailureEffect(error) !== 'none') recordInput(session, [clearReceipt(requested)]); + throw error; + } + recordInput(session, [clearReceipt(used || undefined)]); + if (requested && used !== requested) { + throw attachFailureEffect( + new WebDriverError( + 'unknown error', + `${description} asked for a ${requested} clear, and the host cleared ${used ? `with the ${used} mechanism` : 'without saying how'}.`, + { + reason: 'input-mechanism-mismatch', + }, + ), + 'applied', + ); + } + return used || undefined; +} + +function clearReceipt(mechanism: DesktopClearMechanism | undefined): DesktopInputMechanism { + return mechanism === 'semantic' ? 'semantic-clear' : mechanism === 'keyboard' ? 'keyboard' : 'unattributed-clear'; +} + +function recordInput(session: DesktopSession, mechanisms: readonly DesktopInputMechanism[]): void { + for (const mechanism of mechanisms) { + session.inputReceipts.set(mechanism, (session.inputReceipts.get(mechanism) ?? 0) + 1); + } +} + +function refuse(message: string, data: Record): Error { + const error = new WebDriverError('unsupported operation', message, data); + return attachFailureEffect(error, 'none'); +} diff --git a/packages/agentic/desktop-driver/src/server/sessionOperations.ts b/packages/agentic/desktop-driver/src/server/sessionOperations.ts index 8f35d36e50..c42b07bc7b 100644 --- a/packages/agentic/desktop-driver/src/server/sessionOperations.ts +++ b/packages/agentic/desktop-driver/src/server/sessionOperations.ts @@ -6,6 +6,7 @@ import { safeToken } from '../operations/text.js'; import { HostUnsupportedError, WebDriverError } from '../protocol/errors.js'; import { withCommandTimeout } from '../protocol/timeouts.js'; import type { DesktopSession, SessionManager } from './SessionManager.js'; +import { assertInputAllowed, clickMechanism, deliverClear, deliverInput, strictClearMechanism } from './sessionInput.js'; import { createOperationStories } from './sessionStories.js'; export type SessionOperationOptions = { @@ -40,12 +41,22 @@ export function createOperationRuntime( const listAccessibilityActions = nativeHost.listAccessibilityActions?.bind(nativeHost); const performAccessibilityAction = nativeHost.performAccessibilityAction?.bind(nativeHost); const features = session.hostInfo.features; + // An accessible session offers no operation that types or moves the pointer, so `act` refuses them before it proves + // anything; the host below enforces the policy again when input is dispatched. + const accessible = session.inputMode === 'accessible'; const host: OperationHost = { activate: (windowId, signal) => nativeHost.activate(windowId, signal), captureElement: (nativeId, signal) => nativeHost.captureElement(nativeId, signal), captureWindow: (windowId, signal) => nativeHost.captureWindow(windowId, signal), - clear: (nativeId, signal) => nativeHost.clear(nativeId, signal), - click: (nativeId, mode, signal) => nativeHost.click(nativeId, mode, signal), + async clear(nativeId, signal) { + const requested = strictClearMechanism(session, 'Clearing an element'); + return deliverClear(session, requested, 'Clearing an element', (mechanism) => nativeHost.clear(nativeId, signal, mechanism)); + }, + async click(nativeId, mode, signal) { + const mechanism = clickMechanism(mode); + assertInputAllowed(session, mechanism, 'Clicking an element'); + return deliverInput(session, [mechanism], () => nativeHost.click(nativeId, mode, signal)); + }, hitTest: (windowId, x, y, signal) => nativeHost.hitTest(windowId, x, y, signal), listAccessibilityActions(nativeId, signal) { if (!listAccessibilityActions) { @@ -65,13 +76,17 @@ export function createOperationRuntime( } return releaseElements(nativeIds, signal); }, - performAccessibilityAction(nativeId, key, signal) { + async performAccessibilityAction(nativeId, key, signal) { if (!performAccessibilityAction) { throw new HostUnsupportedError(`Target "${session.target.id}" does not perform accessibility actions.`); } - return performAccessibilityAction(nativeId, key, signal); + assertInputAllowed(session, 'accessibility-action', 'Performing an accessibility action'); + return deliverInput(session, ['accessibility-action'], () => performAccessibilityAction(nativeId, key, signal)); + }, + async sendKeys(nativeId, text, signal) { + assertInputAllowed(session, 'keyboard', 'Typing into an element'); + return deliverInput(session, ['keyboard'], () => nativeHost.sendKeys(nativeId, text, signal)); }, - sendKeys: (nativeId, text, signal) => nativeHost.sendKeys(nativeId, text, signal), snapshot: (nativeId, signal) => nativeHost.snapshot(nativeId, signal), ...(subscribe ? { subscribe } : {}), }; @@ -87,8 +102,8 @@ export function createOperationRuntime( accessibilityClick: features.accessibilityClick, boundedObservation: features.boundedObservation === true && !!observe && !!releaseElements, elementCapture: features.elementScreenshot, - keyboard: features.keyboard, - physicalClick: features.physicalClick, + keyboard: features.keyboard && !accessible, + physicalClick: features.physicalClick && !accessible, stories: !!session.target.storyOrchestrator, windowCapture: features.screenshot, }, diff --git a/packages/agentic/desktop-driver/src/testing/scriptedNativeHelper.ts b/packages/agentic/desktop-driver/src/testing/scriptedNativeHelper.ts new file mode 100644 index 0000000000..0e83534932 --- /dev/null +++ b/packages/agentic/desktop-driver/src/testing/scriptedNativeHelper.ts @@ -0,0 +1,72 @@ +import { EventEmitter } from 'node:events'; +import { PassThrough } from 'node:stream'; + +import { encodeJsonFrame, NativeFrameDecoder } from '../hosts/native/framing.js'; +import type { NativeHostJsonMessage } from '../native/types.js'; + +export type ScriptedNativeRequest = Extract; +export type ScriptedNativeSend = (message: NativeHostJsonMessage) => void; + +class ScriptedNativeChild extends EventEmitter { + readonly stdin = new PassThrough(); + readonly stdout = new PassThrough(); + readonly stderr = new PassThrough(); + exitCode: number | null = null; + signalCode: NodeJS.Signals | null = null; + + kill(): boolean { + this.finish(); + return true; + } + + finish(): void { + this.exitCode = 0; + setImmediate(() => { + this.emit('exit', 0, null); + this.emit('close', 0, null); + }); + } +} + +/** + * Scripts the next native helper a host spawns. The helper says hello with `features`, answers each request through + * `handler`, and acknowledges `dispose` and exits. Pass the test's mocked `spawn`; the returned list collects requests. + */ +export function scriptNativeHelper( + spawn: { mockImplementationOnce(implementation: () => unknown): unknown }, + handler: (message: ScriptedNativeRequest, send: ScriptedNativeSend) => void, + features: readonly string[] = [], +): ScriptedNativeRequest[] { + const child = new ScriptedNativeChild(); + const decoder = new NativeFrameDecoder(); + const received: ScriptedNativeRequest[] = []; + const send: ScriptedNativeSend = (message) => void child.stdout.write(encodeJsonFrame(message)); + child.stdin.on('data', (chunk: Buffer) => decoder.write(chunk)); + decoder.on('json', (message) => { + if (message.type !== 'request') { + return; + } + if (message.command === 'dispose') { + send({ id: message.id, result: {}, type: 'response' }); + child.finish(); + return; + } + received.push(message); + handler(message, send); + }); + spawn.mockImplementationOnce(() => { + setImmediate(() => + send({ + architecture: 'arm64', + buildId: 'build', + features: [...features], + protocol: { major: 1, minor: 0 }, + provider: 'macos', + sourceDigest: 'source', + type: 'hello', + }), + ); + return child; + }); + return received; +} diff --git a/packages/agentic/desktop-driver/src/wdio/DesktopWebdriver.test.ts b/packages/agentic/desktop-driver/src/wdio/DesktopWebdriver.test.ts index 6bf0af6b64..60db490590 100644 --- a/packages/agentic/desktop-driver/src/wdio/DesktopWebdriver.test.ts +++ b/packages/agentic/desktop-driver/src/wdio/DesktopWebdriver.test.ts @@ -106,6 +106,26 @@ describe('sanctioned WebdriverIO API', () => { fs.rmSync(artifactsRoot, { force: true, recursive: true }); } }, 60_000); + + test('opens a session with an input mode, and performs a named accessibility action through an element command', async () => { + const harness = await createDesktopDriverTestHarness({ features: { accessibilityActions: true, performAccessibilityAction: true } }); + harness.host.setElementActions('button', ['Increment', 'Decrement']); + try { + const response = await runContract(harness.server.url, harness.target.id, os.tmpdir(), 'input.contract.cjs'); + expect(response).toEqual({ + attachedInputMode: 'accessible', + input: { mode: 'accessible', receipts: { 'accessibility-action': 1, 'semantic-clear': 1 } }, + inputMode: 'accessible', + typed: expect.stringContaining('never delivers'), + }); + expect(harness.host.actions).toContainEqual({ elementId: 'button', name: 'Increment', type: 'accessibility-action' }); + expect(harness.host.actions).toContainEqual({ elementId: 'input', type: 'clear', mechanism: 'semantic' }); + expect(harness.host.actions.some((action) => action.type === 'send-keys')).toBe(false); + expect(harness.host.actions.filter((action) => action.type === 'clear')).toHaveLength(1); + } finally { + await harness.close(); + } + }, 60_000); }); function runContract( diff --git a/packages/agentic/desktop-driver/src/wdio/DesktopWebdriver.ts b/packages/agentic/desktop-driver/src/wdio/DesktopWebdriver.ts index 731532b6fd..dc8e7e257f 100644 --- a/packages/agentic/desktop-driver/src/wdio/DesktopWebdriver.ts +++ b/packages/agentic/desktop-driver/src/wdio/DesktopWebdriver.ts @@ -2,11 +2,12 @@ import { inspect } from 'node:util'; import { attach, remote } from 'webdriverio'; +import type { DesktopInputMode, DesktopSessionInput } from '../authoring/input.js'; import type { DesktopStoryRunResult, DesktopStoryTestResult } from '../authoring/results.js'; import type { DesktopStoryExpectation } from '../authoring/storyTests.js'; import type { DesktopHostFeatures } from '../host/types.js'; import { ArtifactManager } from '../artifacts/ArtifactManager.js'; -import { createDesktopDriverClient, DesktopSessionClient } from '../client/DesktopDriverClient.js'; +import { createDesktopDriverClient, DesktopElementClient, DesktopSessionClient } from '../client/DesktopDriverClient.js'; import type { DesktopClickMode, DesktopPlatformName } from '../protocol/types.js'; import { assertDesktopExpectation, runDesktopStoryTests } from '../runner/StoryTestRunner.js'; import type { DesktopStoryTestSelection } from '../runner/StoryTestRunner.js'; @@ -19,18 +20,29 @@ declare global { 'furn:clickMode'?: 'accessibility' | 'auto' | 'physical'; 'furn:endpoint'?: 'macos' | 'win32' | 'windows'; 'furn:features'?: Readonly; + 'furn:inputMode'?: 'accessible' | 'compatibility' | 'physical'; 'furn:launchMode'?: 'attach' | 'launch'; 'furn:storybook'?: boolean; 'furn:target'?: string; } interface Browser { + /** The session's input policy and the input it has delivered so far, by mechanism. */ + desktopInput(): Promise; desktopListStories(): Promise; desktopOpenStory(storyId: string, runId?: string): Promise; desktopResetStory(storyId: string, runId?: string): Promise; desktopExpect(expectation: DesktopStoryExpectation): Promise; desktopRunStoryTests(options?: DesktopWebdriverRunOptions): Promise; } + + interface Element { + /** + * Performs the one accessibility action the element publishes under this name, such as `increment`, without + * moving the pointer or pressing a key. Declare `perform-accessibility-action` in a case that calls it. + */ + desktopPerformAccessibilityAction(name: string): Promise; + } } } @@ -38,6 +50,8 @@ export type DesktopWebdriverOptions = { clickMode?: DesktopClickMode; /** Supervisor-provided credential. Never put it in capabilities, command lines, or logs. */ grant?: string; + /** The input the session may deliver. A strict mode pins the click mode. Defaults to `compatibility`. */ + inputMode?: DesktopInputMode; launchMode?: 'attach' | 'launch'; logLevel?: 'debug' | 'error' | 'info' | 'silent' | 'trace' | 'warn'; platformName: DesktopPlatformName; @@ -84,6 +98,11 @@ export class DesktopWebdriverSession { return assertDesktopExpectation(this.session, expectation); } + /** The session's input policy and what it has delivered so far. */ + input(): Promise { + return this.session.getInput(); + } + async runStoryTests(options: DesktopWebdriverRunOptions = {}): Promise { const manifest = await this.session.getStoryManifest(); const endpoint = this.session.capabilities['furn:endpoint']; @@ -122,6 +141,7 @@ export async function connectDesktopWebdriver(options: DesktopWebdriverOptions): }, { 'furn:clickMode': options.clickMode ?? 'auto', + ...(options.inputMode === undefined ? {} : { 'furn:inputMode': options.inputMode }), 'furn:launchMode': options.launchMode ?? 'launch', 'furn:target': options.targetId, }, @@ -235,7 +255,28 @@ export function installDesktopWebdriverCommands( }, { attachToElement: true }, ); + browser.overwriteCommand( + 'setValue', + async function (this: WebdriverIO.Element, originalCommand, value: string | number) { + const inputMode = boundSessions.get(browser)!.session.capabilities['furn:inputMode']; + if (inputMode === 'accessible') { + throw new Error(`setValue would deliver keyboard input, which a session with input mode "${inputMode}" never delivers.`); + } + return originalCommand(value); + }, + { attachToElement: true }, + ); + browser.addCommand( + 'desktopPerformAccessibilityAction', + async function (this: WebdriverIO.Element, name: string) { + const id = await nativeElementId(this); + if (!id) throw new Error('The element was not found, so it has no accessibility action to perform.'); + await new DesktopElementClient(boundSessions.get(browser)!.session, id).performAccessibilityAction(name); + }, + { attachToElement: true }, + ); browser.addCommand('desktopExpect', (expectation: DesktopStoryExpectation) => boundSessions.get(browser)!.expect(expectation)); + browser.addCommand('desktopInput', () => boundSessions.get(browser)!.input()); installedBrowsers.add(browser); } if (browser.capabilities['furn:storybook'] && !installedStoryBrowsers.has(browser)) { diff --git a/packages/agentic/desktop-driver/src/wdio/input.contract.cjs b/packages/agentic/desktop-driver/src/wdio/input.contract.cjs new file mode 100644 index 0000000000..0bcd2caccc --- /dev/null +++ b/packages/agentic/desktop-driver/src/wdio/input.contract.cjs @@ -0,0 +1,34 @@ +const path = require('node:path'); +const { pathToFileURL } = require('node:url'); + +async function main() { + const moduleUrl = pathToFileURL(path.resolve(__dirname, '..', '..', 'lib', 'wdio', 'index.js')).href; + const { attachDesktopWebdriver, connectDesktopWebdriver } = await import(moduleUrl); + const [url, targetId] = process.argv.slice(2); + const desktop = await connectDesktopWebdriver({ inputMode: 'accessible', platformName: 'windows', targetId, url }); + try { + const worker = await attachDesktopWebdriver({ url, sessionId: desktop.browser.sessionId, capabilities: desktop.browser.capabilities }); + await (await worker.browser.$('~button-primary')).desktopPerformAccessibilityAction('Increment'); + const input = await worker.browser.$('~input-name'); + await input.clearValue(); + const typed = await input.setValue('text').then( + () => 'typed', + (error) => error.message, + ); + process.stdout.write( + JSON.stringify({ + attachedInputMode: worker.browser.capabilities['furn:inputMode'], + input: await worker.browser.desktopInput(), + inputMode: desktop.browser.capabilities['furn:inputMode'], + typed, + }), + ); + } finally { + await desktop.delete(); + } +} + +main().catch((error) => { + process.stderr.write(error.stack ?? error.message); + process.exitCode = 1; +}); diff --git a/packages/agentic/desktop-test/references/commands.md b/packages/agentic/desktop-test/references/commands.md index 817553b4bc..abaa38af42 100644 --- a/packages/agentic/desktop-test/references/commands.md +++ b/packages/agentic/desktop-test/references/commands.md @@ -64,8 +64,9 @@ rerunning `prep`. See [platform setup](platforms.md). `test --input accessible|physical` runs one input lane. `DESKTOP_INPUT` supplies the lane when the flag is absent, and an empty or unknown value is an error. The `accessible` lane starts the native host with keyboard, physical click, and wheel -disabled and runs the tests that need no physical input; the `physical` lane -runs tests that need it plus those that need neither. Tests declare which with +disabled and requests a matching session policy; the `physical` lane requests a +physical session policy and runs tests that need physical input plus those that +need neither. Tests declare which with the [`./jasmine` helpers](wdio-suites.md#declare-input-and-scope-in-jasmine-specs). A lane requires a Jasmine WDIO suite (or the Storybook suite, whose integration applies the lane itself) and rejects `--list`, which resolves spec files without @@ -75,7 +76,8 @@ lane run is judged. An imported integration that serves its own driver receives the lane and owns masking that driver. A host supplied through `hostFactory` is not masked by the -harness, so the suite verifies the mask before running anything. +harness, so the suite verifies the mask before running anything. The strict +session policy independently refuses input outside the selected lane. The Storybook `csf-wdio` suite does not support generic `--list`. Discover story IDs with the Storybook integration's manifest/CLI, then select authored tests diff --git a/packages/agentic/desktop-test/references/reporting.md b/packages/agentic/desktop-test/references/reporting.md index ad7b0176ae..1c0c7f64f1 100644 --- a/packages/agentic/desktop-test/references/reporting.md +++ b/packages/agentic/desktop-test/references/reporting.md @@ -78,9 +78,16 @@ holds every selected test to its declared outcome. `run.json` carries the lane a - `masking` (accessible only): `host-masked` when this command started the host with keyboard, physical click, and wheel disabled, or `verified-masked` when it attached to a host and verified that mask before running anything. -- `policyEnforced`: always `false`. The report records what was requested and how - the host was masked; it never claims what input a test delivered, so a physical - lane is labelled a pilot. +- `policyEnforced`: `true` when each worker session confirms its requested + `furn:inputMode`; older reports may record `false`. The session refuses input + outside the lane before dispatch. `masking` is an additional host-level + safeguard, not the policy enforcement mechanism. + +Each executed test attempt may include `input` in `run.json`: `receipts` counts +the mechanisms the session delivered, `undeclared` lists mechanisms the case +did not declare and therefore failed on, and `unused` lists declared input +requirements not exercised. These receipts describe deliveries, not whether an +application changed state or whether a platform/lane is qualified. Tests that a lane, a `platforms` list, or a `scope` leaves out are not tests, so they are never reported as skipped. A producer announces them in `selection` @@ -105,12 +112,12 @@ A lane run promised to execute its whole selection, so its verdict is stricter: its existing verdict. The plain and GitHub summaries show the lane and exclusions on one line, such as -`Selection: input accessible (host-masked) from --input; excluded 4 by platform, 0 by scope, 12 by input`. +`Selection: input accessible (policy-enforced, host-masked) from --input; excluded 4 by platform, 0 by scope, 12 by input`. The JUnit file records the lane as the properties `desktop.input`, `desktop.input.label`, and `desktop.input.source`, the exclusions as `desktop.excluded.platform`, `desktop.excluded.scope`, and `desktop.excluded.input`, -and `desktop.declaredUnsupported` and `desktop.quarantined` when present. The label -is `accessible (host-masked)`, `accessible (verified-masked)`, or `physical pilot`. +and `desktop.declaredUnsupported` and `desktop.quarantined` when present. The label includes `policy-enforced` when the session confirmed that mode; +`physical pilot` still does not claim qualification. A delegated command, which inherits its parent's collector, sends its exclusions to the parent but not its lane, so the parent's `run.json` has no `lane`. The command diff --git a/packages/agentic/desktop-test/references/wdio-suites.md b/packages/agentic/desktop-test/references/wdio-suites.md index fa80879ec3..d169d5c40f 100644 --- a/packages/agentic/desktop-test/references/wdio-suites.md +++ b/packages/agentic/desktop-test/references/wdio-suites.md @@ -93,8 +93,10 @@ meet is excluded the same way and reported unsupported. click in the physical lane. A `click` requirement follows it. Without a lane the session uses `auto`, which lets the host choose. - **Untrustworthy selections fail closed.** A malformed selection, an unverifiable - Jasmine environment, or an accessible lane on a host that does not report keyboard, - physical click, and wheel disabled disables every test and fails the run. + Jasmine environment, or a worker session that does not confirm the requested + `furn:inputMode` disables every test and fails the run. An accessible lane also + requires its host to report keyboard, physical click, and wheel disabled; the + supervisor applies these masks to its own native host. - **A bad declaration never throws.** Jasmine only warns when a spec file fails to load, so a throw would silently drop every test after it. An invalid declaration registers a failing test that reports the problem. @@ -103,5 +105,6 @@ meet is excluded the same way and reported unsupported. - `desktopIt` takes no timeout argument; the suite's `timeoutMs` applies. The input-call lint of Storybook cases does not apply here. Page objects and helpers -hide the input a test sends, so the accessible lane's host mask is the safeguard that -fails a test that reaches for physical input. +hide the input a test sends, so the host mask and session policy reject +undeclared physical input. The reporter records mechanism receipts and fails a +declared case if it delivered input outside its `requires`. diff --git a/packages/agentic/desktop-test/src/cli.ts b/packages/agentic/desktop-test/src/cli.ts index 83e746f174..659800eb62 100644 --- a/packages/agentic/desktop-test/src/cli.ts +++ b/packages/agentic/desktop-test/src/cli.ts @@ -175,7 +175,8 @@ export async function runDesktopTestCli(argv = process.argv): Promise { kind: action === 'test' ? 'tests' : 'lifecycle', optional: suite?.kind === 'wdio' && suite.optional, context: { endpoint: target.endpoint, renderer: target.host.renderer }, - ...(requestedLane ? { lane: createDesktopInputLane(requestedLane.input, requestedLane.source) } : {}), + // The supervisor starts the driver, and starts it masked for an accessible lane. + ...(requestedLane ? { lane: createDesktopInputLane(requestedLane.input, requestedLane.source, 'host-masked') } : {}), reporterMode, eventLevel, suppressFinalOutput: action === 'serve' ? () => ready : false, diff --git a/packages/agentic/desktop-test/src/contracts.contract.cjs b/packages/agentic/desktop-test/src/contracts.contract.cjs index e6203aa2e7..e00218fbf8 100644 --- a/packages/agentic/desktop-test/src/contracts.contract.cjs +++ b/packages/agentic/desktop-test/src/contracts.contract.cjs @@ -1748,18 +1748,18 @@ async function main() { input: 'accessible', source: 'option', masking: 'host-masked', - policyEnforced: false, + policyEnforced: true, }); ran(invoke(['test', '--suite', 'csf'], { DESKTOP_INPUT: 'physical' }), 'physical', { input: 'physical', source: 'environment', - policyEnforced: false, + policyEnforced: true, }); ran(invoke(['test', '--suite', 'csf', '--input', 'accessible'], { DESKTOP_INPUT: 'physical' }), 'accessible', { input: 'accessible', source: 'option', masking: 'host-masked', - policyEnforced: false, + policyEnforced: true, }); const unlaned = invoke(['test', '--suite', 'csf']); assert.equal(unlaned.status, 0, unlaned.diagnostics); @@ -1789,7 +1789,14 @@ async function main() { const { FakeDesktopHost } = await import('@fluentui-react-native/desktop-driver/testing'); const laneMode = mode.startsWith('lane-'); const retried = mode === 'retry' || mode === 'lane-physical'; - const lane = laneMode ? createDesktopInputLane(mode === 'lane-physical' ? 'physical' : 'accessible', 'option') : undefined; + // Like the CLI, the fixture starts the host itself, masked for the accessible lane. + const lane = laneMode + ? createDesktopInputLane( + mode === 'lane-physical' ? 'physical' : 'accessible', + 'option', + mode === 'lane-accessible' ? 'host-masked' : undefined, + ) + : undefined; // The lane fixtures import the Jasmine adapter by file URL from the environment, so no value is spliced into source. if (laneMode) process.env.DESKTOP_JASMINE_URL = lib('jasmine/index.js'); const run = reporter({ context: { endpoint: 'win32', renderer: 'paper' }, ...(lane ? { lane } : {}) }); @@ -1844,6 +1851,7 @@ async function main() { " desktopIt('runs when nothing is declared', () => {});", " desktopIt('clicks as the lane does', { requires: ['click'] }, async () => { await (await browser.$('~button-primary')).click(); });", " desktopIt('clicks accessibly', { requires: ['accessibility-click'] }, async () => { await (await browser.$('~button-primary')).click(); });", + " desktopIt('clicks without declaring it', async () => { await (await browser.$('~button-primary')).click(); });", " desktopIt('physical only', { requires: ['keyboard'] }, mustNotRun('an input-excluded case'));", " desktopIt('needs what the host lacks', { requires: ['accessibility-actions'] }, mustNotRun('a case the host cannot support'));", '});', @@ -1983,18 +1991,24 @@ async function main() { } } else if (mode === 'lane-accessible') { const text = JSON.stringify(result); - // The host cannot support one case. A lane run holds that against the run instead of quietly skipping it. - assert(failure && /incomplete/.test(failure.message), `${failure?.stack}\n${text}\n${err.text()}`); - assert.notEqual(result.status, 'passed', text); + // One case delivered input it never declared, and the host cannot support another, which a lane run also holds + // against the run instead of quietly skipping it. + assert(failure, `${text}\n${err.text()}`); + assert.equal(result.status, 'failed', text); assert.equal(launches, 1); assert.equal(result.counts.passed, 3, text); - assert.equal(result.counts.failed, 0, text); + // The policy allowed the click, and its receipt failed the case that never declared it. + assert.equal(result.counts.failed, 1, text); + const undeclared = attemptOf(result, 'clicks without declaring it'); + assert.deepEqual(undeclared.input, { receipts: { 'accessibility-click': 1 }, undeclared: ['accessibility-click'] }, text); + assert.match(undeclared.failure.error.message, /delivered input it does not declare: accessibility-click/, text); + assert.deepEqual(attemptOf(result, 'clicks as the lane does').input, { receipts: { 'accessibility-click': 1 } }, text); assert.equal(result.counts.unsupported, 1, text); assert.equal(result.counts.declaredUnsupported, 0, text); assert.deepEqual(result.selection.excluded, { platform: 0, scope: 0, input: 1 }, text); - assert.deepEqual(result.lane, { input: 'accessible', source: 'option', masking: 'host-masked', policyEnforced: false }); + assert.deepEqual(result.lane, { input: 'accessible', source: 'option', masking: 'host-masked', policyEnforced: true }); const clicks = host.actions.filter((action) => action.type === 'click'); - assert(clicks.length === 2 && clicks.every((action) => action.mode === 'accessibility'), JSON.stringify(host.actions)); + assert(clicks.length === 3 && clicks.every((action) => action.mode === 'accessibility'), JSON.stringify(host.actions)); } else { assert.equal(failure, undefined, `${failure?.stack}\n${err.text()}\n${out.text()}`); assert.equal(result.status, 'passed', JSON.stringify(result)); @@ -2011,7 +2025,9 @@ async function main() { assert.equal(result.counts.declaredUnsupported, 1, text); // Both attempts announced the same exclusions, and a retried group still counts them once. assert.deepEqual(result.selection.excluded, { platform: 2, scope: 1, input: 1 }, text); - assert.deepEqual(result.lane, { input: 'physical', source: 'option', policyEnforced: false }); + assert.deepEqual(result.lane, { input: 'physical', source: 'option', policyEnforced: true }); + assert.deepEqual(attemptOf(result, 'clicks as the lane does').input, { receipts: { 'physical-click': 1 } }, text); + assert.deepEqual(attemptOf(result, 'needs the keyboard').input, { receipts: {}, unused: ['keyboard'] }, text); const clicks = host.actions.filter((action) => action.type === 'click'); assert(clicks.length > 0 && clicks.every((action) => action.mode === 'physical'), JSON.stringify(host.actions)); } @@ -2025,6 +2041,13 @@ async function main() { } } +/** The last attempt of the one test whose title ends with `title`. */ +function attemptOf(result, title) { + const matches = result.tests.filter((test) => test.title.endsWith(title)); + assert.equal(matches.length, 1, `expected one test titled "${title}"\n${JSON.stringify(result.tests)}`); + return matches[0].attempts.at(-1); +} + main().catch((error) => { process.stderr.write(error.stack ?? String(error)); process.exitCode = 1; diff --git a/packages/agentic/desktop-test/src/jasmine/adapter.test.ts b/packages/agentic/desktop-test/src/jasmine/adapter.test.ts index 5b0764a76c..5a9ca56b4e 100644 --- a/packages/agentic/desktop-test/src/jasmine/adapter.test.ts +++ b/packages/agentic/desktop-test/src/jasmine/adapter.test.ts @@ -206,3 +206,66 @@ describe('desktopDescribe', () => { expect(registered.filter(({ kind }) => kind === 'it' || kind === 'xit').map(({ kind }) => kind)).toEqual(['it']); }); }); + +describe('desktopIt input attribution', () => { + function attributed(lane?: 'accessible' | 'physical') { + const registry = createDesktopJasmineRegistry(laneEnvironment(lane)); + const jasmine = fakeJasmine(); + const session = { receipts: { keyboard: 3 } as Record | undefined }; + const deliver = (mechanism: string, count = 1) => { + session.receipts = { ...session.receipts, [mechanism]: (session.receipts?.[mechanism] ?? 0) + count }; + }; + const { desktopIt } = createDesktopJasmine(jasmine.api, registry, async () => session.receipts); + const run = (name: string, context: object = {}) => jasmine.registered.find((spec) => spec.name === name)!.body!.call(context); + return { registry, session, deliver, desktopIt, run }; + } + + test('records only what the body delivered, and the declared input it never used', async () => { + const { deliver, desktopIt, registry, run } = attributed('physical'); + desktopIt('types', { requires: ['keyboard', 'physical-click'] }, () => deliver('keyboard', 2)); + + await run('types'); + + expect(registry.entries.get('typesspec0')).toMatchObject({ input: { receipts: { keyboard: 2 }, unused: ['physical-click'] } }); + }); + + test('fails a body that delivered input nothing declares, once it has run', async () => { + const { deliver, desktopIt, registry, run } = attributed(); + desktopIt('sneaks', async () => deliver('wheel')); + + await expect(run('sneaks')).rejects.toThrow(/The test delivered input it does not declare: wheel \(declare wheel\)\./); + expect(registry.entries.get('sneaksspec0')).toMatchObject({ input: { receipts: { wheel: 1 }, undeclared: ['wheel'] } }); + }); + + test('a failed body keeps its own error and its receipts, and reports nothing unused', async () => { + const { deliver, desktopIt, registry, run } = attributed('physical'); + desktopIt('breaks', { requires: ['keyboard', 'wheel'] }, async () => { + deliver('keyboard'); + throw new Error('assertion'); + }); + + await expect(run('breaks')).rejects.toThrow('assertion'); + expect(registry.entries.get('breaksspec0')).toMatchObject({ input: { receipts: { keyboard: 1 } } }); + expect(registry.entries.get('breaksspec0')).not.toHaveProperty('input.unused'); + }); + + test('outside a lane, a click declaration accounts for whichever click the host chose', async () => { + const { deliver, desktopIt, run } = attributed(); + desktopIt('clicks', { requires: ['click'] }, () => deliver('physical-click')); + + await expect(run('clicks')).resolves.toBeUndefined(); + }); + + test('runs the body as it is, with its context, when the session reports no receipts', async () => { + const { desktopIt, registry, run, session } = attributed(); + session.receipts = undefined; + const context = { shared: true }; + desktopIt('plain', function (this: unknown) { + expect(this).toBe(context); + }); + + await run('plain', context); + + expect(registry.entries.get('plainspec0')).not.toHaveProperty('input'); + }); +}); diff --git a/packages/agentic/desktop-test/src/jasmine/adapter.ts b/packages/agentic/desktop-test/src/jasmine/adapter.ts index 9139684e15..1762fe0473 100644 --- a/packages/agentic/desktop-test/src/jasmine/adapter.ts +++ b/packages/agentic/desktop-test/src/jasmine/adapter.ts @@ -1,13 +1,18 @@ import { + compareDesktopInputReceipts, decideDesktopCase, + describeUndeclaredDesktopInput, mergeDesktopCaseOptions, + resolveDesktopClickMode, + subtractDesktopInputReceipts, validateDesktopCaseOptions, validateDesktopGroupOptions, } from '@fluentui-react-native/desktop-driver/authoring'; -import type { DesktopCaseOptions, ResolvedDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopCaseOptions, DesktopInputReceipts, ResolvedDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; import { desktopSpecIdentifier } from './registry.js'; import type { DesktopJasmineRegistry, DesktopSpecEntry } from './registry.js'; +/** WDIO calls a spec body without Jasmine's `done`, so a body finishes by returning, or by settling the promise it returns. */ export type DesktopSpecBody = (done: (error?: unknown) => void) => unknown; export type DesktopGroupBody = (this: unknown) => void; /** A group scopes the cases inside it. A gap describes one case, so a group cannot declare one. */ @@ -20,13 +25,21 @@ export type DesktopJasmineApi = { xit(name: string): unknown; }; +/** Reads the receipts of the session the specs drive, or nothing when the session cannot report them. */ +export type DesktopInputReader = () => Promise; + +type RunEntry = Extract; + /** * Declares tests with their input modality and the places they run, and registers them with Jasmine. * * `desktopIt` and `desktopDescribe` never throw for a bad declaration: Jasmine only warns when a spec file fails to load, * so a throw would silently drop every test after it. A bad declaration becomes a failing spec that reports the problem. + * + * With `readInput`, each executed body is attributed: the input its session delivered while the body ran is compared with + * the declaration, and a mechanism nothing declares fails the test. */ -export function createDesktopJasmine(api: DesktopJasmineApi, registry: DesktopJasmineRegistry) { +export function createDesktopJasmine(api: DesktopJasmineApi, registry: DesktopJasmineRegistry, readInput?: DesktopInputReader) { function record(spec: unknown, name: string, entry: DesktopSpecEntry): void { const identifier = desktopSpecIdentifier(spec as { description?: unknown; id?: unknown } | undefined); if (identifier === undefined) { @@ -46,6 +59,35 @@ export function createDesktopJasmine(api: DesktopJasmineApi, registry: DesktopJa ); } + function attributed(body: DesktopSpecBody, entry: RunEntry, read: DesktopInputReader): DesktopSpecBody { + return async function (this: unknown, ...args: unknown[]) { + delete entry.input; + const before = await read(); + if (!before) return body.apply(this, args as Parameters); + let failure: { error: unknown } | undefined; + try { + await body.apply(this, args as Parameters); + } catch (error) { + failure = { error }; + } + try { + const after = await read(); + if (after) { + const receipts = subtractDesktopInputReceipts(after, before); + // The session's click mode is the lane's, so a click the host chose is judged the way it was delivered. + const clickMode = resolveDesktopClickMode(undefined, 'auto', registry.selection?.lane); + const { undeclared, unused } = compareDesktopInputReceipts(entry.descriptor, receipts, clickMode); + entry.input = { receipts, ...(undeclared.length ? { undeclared } : {}), ...(!failure && unused.length ? { unused } : {}) }; + } + } catch (error) { + // A failed test keeps its own error; the receipts were only evidence for it. + if (!failure) throw error; + } + if (failure) throw failure.error; + if (entry.input?.undeclared) throw new Error(describeUndeclaredDesktopInput(entry.input.undeclared)); + }; + } + function desktopIt(name: string, optionsOrBody: DesktopCaseOptions | DesktopSpecBody, maybeBody?: DesktopSpecBody): void { const hasOptions = typeof optionsOrBody !== 'function'; const body = hasOptions ? maybeBody : optionsOrBody; @@ -64,7 +106,8 @@ export function createDesktopJasmine(api: DesktopJasmineApi, registry: DesktopJa return; } if (decision.kind === 'run') { - record(api.it(name, body), name, { kind: 'run', descriptor }); + const entry: RunEntry = { kind: 'run', descriptor }; + record(api.it(name, readInput ? attributed(body, entry, readInput) : body), name, entry); return; } // A placeholder keeps the spec out of Jasmine's hooks and body without reporting it as a skipped test. diff --git a/packages/agentic/desktop-test/src/jasmine/index.ts b/packages/agentic/desktop-test/src/jasmine/index.ts index b602b7c452..4fcb7ffdf9 100644 --- a/packages/agentic/desktop-test/src/jasmine/index.ts +++ b/packages/agentic/desktop-test/src/jasmine/index.ts @@ -1,7 +1,7 @@ import { createDesktopJasmine } from './adapter.js'; -import type { DesktopGroupBody, DesktopGroupOptions, DesktopJasmineApi, DesktopSpecBody } from './adapter.js'; +import type { DesktopGroupBody, DesktopGroupOptions, DesktopInputReader, DesktopJasmineApi, DesktopSpecBody } from './adapter.js'; import { desktopJasmineRegistry } from './registry.js'; -import type { DesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopCaseOptions, DesktopSessionInput } from '@fluentui-react-native/desktop-driver/authoring'; export type { DesktopCaseOptions, @@ -18,7 +18,12 @@ const jasmineApi: DesktopJasmineApi = { it: (name, body) => globals().it(name, body), xit: (name) => globals().xit(name), }; -const adapter = () => createDesktopJasmine(jasmineApi, desktopJasmineRegistry()); +// The desktop commands are installed on the session before any spec runs, so a body is attributed whenever they exist. +const readInput: DesktopInputReader = async () => { + const browser = (globalThis as { browser?: { desktopInput?: () => Promise } }).browser; + return browser?.desktopInput ? (await browser.desktopInput()).receipts : undefined; +}; +const adapter = () => createDesktopJasmine(jasmineApi, desktopJasmineRegistry(), readInput); /** * Declares a Jasmine test with the input it needs and where it runs. The descriptor decides whether the test is part of a diff --git a/packages/agentic/desktop-test/src/jasmine/registry.ts b/packages/agentic/desktop-test/src/jasmine/registry.ts index d927d700f1..1a66b45978 100644 --- a/packages/agentic/desktop-test/src/jasmine/registry.ts +++ b/packages/agentic/desktop-test/src/jasmine/registry.ts @@ -4,14 +4,17 @@ import type { DesktopRequirement, ResolvedDesktopCaseOptions, } from '@fluentui-react-native/desktop-driver/authoring'; -import type { SelectionExclusions } from '../reporting/protocol.js'; +import type { SelectionExclusions, TestInputReport } from '../reporting/protocol.js'; import { desktopTestSelectionEnvironment, parseDesktopTestSelection } from './selection.js'; import type { DesktopTestSelection } from './selection.js'; /** What the adapter decided for a spec it registered. */ export type DesktopSpecEntry = - /** Executes, unless a preflight found that the host cannot meet what it requires. */ - | { kind: 'run'; descriptor: ResolvedDesktopCaseOptions; missing?: readonly DesktopRequirement[] } + /** + * Executes, unless a preflight found that the host cannot meet what it requires. `input` is what its body delivered, + * once it ran in a session that reports receipts. + */ + | { kind: 'run'; descriptor: ResolvedDesktopCaseOptions; missing?: readonly DesktopRequirement[]; input?: TestInputReport } /** Registered as a placeholder that never runs and is counted, not reported as a test. */ | { kind: 'excluded'; reason: DesktopExclusionReason } /** Registered as a placeholder that is reported as unsupported, with the reason its author gave. */ diff --git a/packages/agentic/desktop-test/src/jasmine/specSelection.test.ts b/packages/agentic/desktop-test/src/jasmine/specSelection.test.ts index 85c105ec72..750ddb4cff 100644 --- a/packages/agentic/desktop-test/src/jasmine/specSelection.test.ts +++ b/packages/agentic/desktop-test/src/jasmine/specSelection.test.ts @@ -70,7 +70,7 @@ describe('selectDesktopSpecs preflight', () => { desktopIt('needs click', { requires: ['physical-click'] }, body); desktopIt('needs nothing', body); - selectDesktopSpecs(registry, environment, { keyboard: false, physicalClick: true }); + selectDesktopSpecs(registry, environment, { keyboard: false, physicalClick: true }, 'physical'); expect([runs('needs keyboard'), runs('needs click'), runs('needs nothing')]).toEqual([false, true, true]); expect(registry.entries.get('needs keyboardspec0')).toMatchObject({ kind: 'run', missing: ['keyboard'] }); @@ -83,7 +83,7 @@ describe('selectDesktopSpecs preflight', () => { desktopIt('needs keyboard', { requires: ['keyboard'] }, body); desktopIt('kept', body); - selectDesktopSpecs(registry, environment, { keyboard: false }); + selectDesktopSpecs(registry, environment, { keyboard: false }, 'physical'); expect([runs('user filtered'), runs('needs keyboard'), runs('kept')]).toEqual([false, false, true]); }); @@ -92,7 +92,7 @@ describe('selectDesktopSpecs preflight', () => { const { configure, desktopIt, environment, registry } = setup('physical'); desktopIt('needs keyboard', { requires: ['keyboard'] }, body); - selectDesktopSpecs(registry, environment, { keyboard: true }); + selectDesktopSpecs(registry, environment, { keyboard: true }, 'physical'); expect(configure).not.toHaveBeenCalled(); }); @@ -100,12 +100,12 @@ describe('selectDesktopSpecs preflight', () => { test('judges a click requirement by what the lane clicks with', () => { const physical = setup('physical'); physical.desktopIt('clicks', { requires: ['click'] }, body); - selectDesktopSpecs(physical.registry, physical.environment, { physicalClick: false, accessibilityClick: true }); + selectDesktopSpecs(physical.registry, physical.environment, { physicalClick: false, accessibilityClick: true }, 'physical'); expect(physical.runs('clicks')).toBe(false); const accessible = setup('accessible'); accessible.desktopIt('clicks', { requires: ['click'] }, body); - selectDesktopSpecs(accessible.registry, accessible.environment, { ...masked, accessibilityClick: true }); + selectDesktopSpecs(accessible.registry, accessible.environment, { ...masked, accessibilityClick: true }, 'accessible'); expect(accessible.runs('clicks')).toBe(true); }); }); @@ -115,24 +115,39 @@ describe('selectDesktopSpecs fail-closed checks', () => { const { desktopIt, environment, registry, runs } = setup('accessible'); desktopIt('fine', body); - expect(() => selectDesktopSpecs(registry, environment, { keyboard: true, physicalClick: false })).toThrow( + expect(() => selectDesktopSpecs(registry, environment, { keyboard: true, physicalClick: false }, 'accessible')).toThrow( 'The accessible input lane needs a host that reports keyboard, wheel disabled, and this one does not.', ); expect(registry.blocked).toBe(true); expect(runs('fine')).toBe(false); }); + test('a lane whose session did not confirm its input policy disables every spec and says why', () => { + const { desktopIt, environment, registry, runs } = setup('accessible'); + desktopIt('fine', body); + + expect(() => selectDesktopSpecs(registry, environment, masked, 'compatibility')).toThrow( + 'The accessible input lane needs a session that enforces the accessible input policy, and the driver reported compatibility.', + ); + expect(registry.blocked).toBe(true); + expect(runs('fine')).toBe(false); + expect(() => selectDesktopSpecs(setup('physical').registry, environment, {})).toThrow( + /physical input policy, and the driver reported none\./, + ); + }); + test('an absent feature report counts as unmasked', () => { - const { environment, registry } = setup('accessible'); + const { desktopIt, environment, registry } = setup('accessible'); + desktopIt('fine', body); - expect(() => selectDesktopSpecs(registry, environment, undefined)).toThrow(/keyboard, physicalClick, wheel disabled/); + expect(() => selectDesktopSpecs(registry, environment, undefined, 'accessible')).toThrow(/keyboard, physicalClick, wheel disabled/); }); test('an accessible lane on a masked host runs', () => { const { desktopIt, environment, registry, runs } = setup('accessible'); desktopIt('fine', body); - selectDesktopSpecs(registry, environment, masked); + selectDesktopSpecs(registry, environment, masked, 'accessible'); expect(registry.blocked).toBe(false); expect(runs('fine')).toBe(true); @@ -142,7 +157,7 @@ describe('selectDesktopSpecs fail-closed checks', () => { const { desktopIt, environment, registry } = setup('physical'); desktopIt('fine', body); - expect(() => selectDesktopSpecs(registry, environment, { keyboard: true })).not.toThrow(); + expect(() => selectDesktopSpecs(registry, environment, { keyboard: true }, 'physical')).not.toThrow(); }); test('a lane refuses specs that were not declared with desktopIt, however many', () => { @@ -152,7 +167,7 @@ describe('selectDesktopSpecs fail-closed checks', () => { for (let index = 0; index < 12; index++) api.it(`spec ${index}`, body); }); - expect(() => selectDesktopSpecs(registry, environment, {})).toThrow( + expect(() => selectDesktopSpecs(registry, environment, {}, 'physical')).toThrow( /12 specs were not declared with desktopIt or desktopDescribe: "legacy spec 0", .*"legacy spec 9" and 2 more\./, ); expect(registry.blocked).toBe(true); @@ -177,7 +192,7 @@ describe('selectDesktopSpecs fail-closed checks', () => { const { environment, registry } = setup('accessible'); registry.errors.push('Jasmine did not identify the spec registered for "ghost", so its selection cannot be verified.'); - expect(() => selectDesktopSpecs(registry, environment, {})).toThrow(/ghost[\s\S]*accessible input lane/); + expect(() => selectDesktopSpecs(registry, environment, {}, 'accessible')).toThrow(/ghost[\s\S]*accessible input lane/); expect(registry.blocked).toBe(true); }); }); diff --git a/packages/agentic/desktop-test/src/jasmine/specSelection.ts b/packages/agentic/desktop-test/src/jasmine/specSelection.ts index ac3c059544..9d0a945a03 100644 --- a/packages/agentic/desktop-test/src/jasmine/specSelection.ts +++ b/packages/agentic/desktop-test/src/jasmine/specSelection.ts @@ -34,9 +34,16 @@ export function selectDesktopSpecs( registry: DesktopJasmineRegistry, environment: DesktopJasmineEnvironment | undefined, features: unknown, + inputMode?: unknown, ): void { const lane = registry.selection?.lane; const problems = [...registry.errors]; + // The driver refuses what the lane's policy forbids only when the session confirmed that policy. + if (lane && inputMode !== lane) { + problems.push( + `The ${lane} input lane needs a session that enforces the ${lane} input policy, and the driver reported ${typeof inputMode === 'string' ? inputMode : 'none'}.`, + ); + } if (lane === 'accessible') { const unmasked = unmaskedAccessibleFeatures(features); if (unmasked.length > 0) { diff --git a/packages/agentic/desktop-test/src/reporting/collector.test.ts b/packages/agentic/desktop-test/src/reporting/collector.test.ts index 3e811e66e6..93a7940295 100644 --- a/packages/agentic/desktop-test/src/reporting/collector.test.ts +++ b/packages/agentic/desktop-test/src/reporting/collector.test.ts @@ -8,7 +8,7 @@ import { DesktopRunReporter } from './collector.js'; import type { DesktopInputLaneReport } from './results.js'; describe('desktop run reporter input lanes', () => { - const accessible: DesktopInputLaneReport = { input: 'accessible', source: 'option', masking: 'host-masked', policyEnforced: false }; + const accessible: DesktopInputLaneReport = { input: 'accessible', source: 'option', masking: 'host-masked', policyEnforced: true }; let artifactsRoot: string; beforeEach(() => { @@ -104,7 +104,9 @@ describe('desktop run reporter input lanes', () => { expect(result.status).toBe('incomplete'); expect(output()).toContain('2 unsupported (1 declared, 0 quarantined)'); - expect(output()).toContain('[desktop] input accessible (host-masked) from --input; excluded 0 by platform, 0 by scope, 3 by input'); + expect(output()).toContain( + '[desktop] input accessible (policy-enforced, host-masked) from --input; excluded 0 by platform, 0 by scope, 3 by input', + ); }); test('the same unsupported case does not fail a compatibility run', async () => { diff --git a/packages/agentic/desktop-test/src/reporting/collector.ts b/packages/agentic/desktop-test/src/reporting/collector.ts index d8ea65525e..719a281cc6 100644 --- a/packages/agentic/desktop-test/src/reporting/collector.ts +++ b/packages/agentic/desktop-test/src/reporting/collector.ts @@ -397,6 +397,9 @@ export class DesktopRunReporter { this.logger.log( `${prefix} ${event.payload.status.toUpperCase()} ${event.payload.title}${event.payload.reason ? `: ${event.payload.reason}` : ''}`, ); + // Reported, never failed: a declaration may cover a path this attempt did not take. + if (event.payload.input?.unused) + this.logger.log(`${prefix} declared input this attempt did not deliver: ${event.payload.input.unused.join(', ')}`); if (failure && firstFailure) this.logger.error(`${prefix} ${errorDiagnostic(failure.error)}`); break; case 'artifact': diff --git a/packages/agentic/desktop-test/src/reporting/index.ts b/packages/agentic/desktop-test/src/reporting/index.ts index 16383d62c2..4765f332aa 100644 --- a/packages/agentic/desktop-test/src/reporting/index.ts +++ b/packages/agentic/desktop-test/src/reporting/index.ts @@ -13,6 +13,7 @@ export type { ReportFailure, SelectionExclusions, TestGap, + TestInputReport, TestStatus, } from './protocol.js'; export type { DesktopEventRecord } from './structured.js'; @@ -31,6 +32,7 @@ export { summarizeDesktopRun, } from './results.js'; export type { + DesktopInputLaneMasking, DesktopInputLaneReport, DesktopInputLaneSource, DesktopRunResult, diff --git a/packages/agentic/desktop-test/src/reporting/lane.test.ts b/packages/agentic/desktop-test/src/reporting/lane.test.ts index f10c320dd9..9c312ff764 100644 --- a/packages/agentic/desktop-test/src/reporting/lane.test.ts +++ b/packages/agentic/desktop-test/src/reporting/lane.test.ts @@ -18,17 +18,18 @@ describe('requested desktop input lane', () => { }); describe('createDesktopInputLane', () => { - test('an accessible lane says how it was masked, and neither lane claims enforcement yet', () => { - expect(createDesktopInputLane('accessible', 'option')).toEqual({ + test('every lane is policy-enforced, and only an accessible lane records host masks', () => { + expect(createDesktopInputLane('accessible', 'option', 'host-masked')).toEqual({ input: 'accessible', source: 'option', masking: 'host-masked', - policyEnforced: false, + policyEnforced: true, }); - expect(createDesktopInputLane('accessible', 'driver', 'verified-masked')).toMatchObject({ - masking: 'verified-masked', - source: 'driver', + expect(createDesktopInputLane('accessible', 'driver')).toEqual({ input: 'accessible', source: 'driver', policyEnforced: true }); + expect(createDesktopInputLane('physical', 'environment', 'host-masked')).toEqual({ + input: 'physical', + source: 'environment', + policyEnforced: true, }); - expect(createDesktopInputLane('physical', 'environment')).toEqual({ input: 'physical', source: 'environment', policyEnforced: false }); }); }); diff --git a/packages/agentic/desktop-test/src/reporting/lane.ts b/packages/agentic/desktop-test/src/reporting/lane.ts index 49064f01e2..968c713059 100644 --- a/packages/agentic/desktop-test/src/reporting/lane.ts +++ b/packages/agentic/desktop-test/src/reporting/lane.ts @@ -25,13 +25,16 @@ export function requestedDesktopInputLane(request: { flag?: string; environment? } /** - * How an accessible lane was masked. `host-masked` means this command started the host with keyboard, physical click and - * wheel disabled. `verified-masked` means it attached to a host and checked the mask before running anything. + * The lane a run holds its tests to. Every runner verifies that each session confirmed the lane's `furn:inputMode` and + * stops otherwise, so a lane is always policy-enforced. Pass `host-masked` when this command also started the host with + * keyboard, physical click and wheel disabled for an accessible lane. */ export function createDesktopInputLane( input: DesktopInputLaneName, source: DesktopInputLaneSource, - masking: 'host-masked' | 'verified-masked' = 'host-masked', + masking?: 'host-masked', ): DesktopInputLaneReport { - return input === 'physical' ? { input, source, policyEnforced: false } : { input, source, masking, policyEnforced: false }; + return input === 'physical' + ? { input, source, policyEnforced: true } + : { input, source, ...(masking ? { masking } : {}), policyEnforced: true }; } diff --git a/packages/agentic/desktop-test/src/reporting/protocol.test.ts b/packages/agentic/desktop-test/src/reporting/protocol.test.ts index 2462042655..315aa96b2f 100644 --- a/packages/agentic/desktop-test/src/reporting/protocol.test.ts +++ b/packages/agentic/desktop-test/src/reporting/protocol.test.ts @@ -28,6 +28,34 @@ describe('declared gaps and selection events', () => { expect(() => validateDesktopEvent(event('test-result', { ...result, status: 'unsupported', gap: 'flaky' }))).toThrow(); }); + test('accepts delivered input, and undeclared input only on a failed result', () => { + const receipts = { 'accessibility-click': 1, keyboard: 2 }; + const failed = { + ...result, + status: 'failed', + failure: { failureId: 'failure', kind: 'test', error: { name: 'Error', message: 'Undeclared input.' } }, + }; + expect(() => validateDesktopEvent(event('test-result', { ...result, status: 'passed', input: { receipts } }))).not.toThrow(); + expect(() => + validateDesktopEvent(event('test-result', { ...result, status: 'passed', input: { receipts, unused: ['wheel'] } })), + ).not.toThrow(); + expect(() => validateDesktopEvent(event('test-result', { ...failed, input: { receipts, undeclared: ['keyboard'] } }))).not.toThrow(); + expect(() => + validateDesktopEvent(event('test-result', { ...result, status: 'passed', input: { receipts, undeclared: ['keyboard'] } })), + ).toThrow('must fail'); + }); + + test.each([ + ['an unknown mechanism', { receipts: { typing: 1 } }], + ['a negative count', { receipts: { keyboard: -1 } }], + ['an empty undeclared list', { receipts: {}, undeclared: [] }], + ['an unknown requirement', { receipts: {}, unused: ['typing'] }], + ['an unknown field', { receipts: {}, typed: 'secret' }], + ])('rejects input with %s', (_name, input) => { + // A passed result needs nothing else, so the input is what fails it. + expect(() => validateDesktopEvent(event('test-result', { ...result, status: 'passed', input }))).toThrow(); + }); + test('accepts exactly the three non-negative integer exclusion counts', () => { expect(() => validateDesktopEvent(event('selection', { excluded: { platform: 1, scope: 0, input: 12 } }))).not.toThrow(); }); diff --git a/packages/agentic/desktop-test/src/reporting/protocol.ts b/packages/agentic/desktop-test/src/reporting/protocol.ts index 524c10f50b..14b2754831 100644 --- a/packages/agentic/desktop-test/src/reporting/protocol.ts +++ b/packages/agentic/desktop-test/src/reporting/protocol.ts @@ -1,4 +1,6 @@ import { randomUUID } from 'node:crypto'; +import { desktopInputMechanisms, desktopRequirements, validateDesktopInputReceipts } from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopInputMechanism, DesktopInputReceipts, DesktopRequirement } from '@fluentui-react-native/desktop-driver/authoring'; export type FailureKind = 'test' | 'infrastructure' | 'cleanup' | 'evidence' | 'reporting'; export type ReportError = { name: string; message: string; stack?: string; cause?: ReportError; errors?: ReportError[] }; @@ -21,6 +23,15 @@ export const testGaps: readonly TestGap[] = ['unsupported', 'quarantine']; /** Cases a producer removed from its selection before any session existed, by the reason they were removed. */ export type SelectionExclusions = { platform: number; scope: number; input: number }; export const selectionExclusionReasons: readonly (keyof SelectionExclusions)[] = ['platform', 'scope', 'input']; +/** + * The input one test attempt delivered, read from its session's receipts, and how it compares with the declaration. + * `undeclared` mechanisms failed the attempt; `unused` declarations are reported only. + */ +export type TestInputReport = { + receipts: DesktopInputReceipts; + undeclared?: DesktopInputMechanism[]; + unused?: DesktopRequirement[]; +}; export type EventContext = { endpoint?: 'macos' | 'windows' | 'win32'; renderer?: 'fabric' | 'paper'; @@ -50,6 +61,7 @@ export type EventData = durationMs: number; reason?: string; gap?: TestGap; + input?: TestInputReport; failure?: ReportFailure; }; } @@ -82,7 +94,7 @@ const payloadFields = { diagnostic: ['level', 'message', 'failure'], output: ['stream', 'text'], 'test-start': ['testId', 'title', 'attempt'], - 'test-result': ['testId', 'title', 'attempt', 'status', 'durationMs', 'reason', 'gap', 'failure'], + 'test-result': ['testId', 'title', 'attempt', 'status', 'durationMs', 'reason', 'gap', 'input', 'failure'], selection: ['excluded', 'key'], artifact: ['path', 'description'], 'producer-finished': ['status'], @@ -191,6 +203,11 @@ export function validateDesktopEvent(value: unknown): asserts value is DesktopEv member(payload.gap, testGaps); if (payload.status !== 'unsupported') throw new TypeError('A declared gap is only valid on an unsupported test result.'); } + if (payload.input !== undefined) { + validateTestInputReport(payload.input); + if (payload.input.undeclared?.length && payload.status !== 'failed') + throw new TypeError('A test that delivered undeclared input must fail.'); + } if (payload.failure !== undefined) validateFailure(payload.failure); } break; @@ -268,6 +285,22 @@ function validateFailure(value: unknown): asserts value is ReportFailure { checkError(failure.error, 0); } +/** Checks the input an attempt reported, in an event or a persisted run, so a malformed one is never shown as evidence. */ +export function validateTestInputReport(value: unknown): asserts value is TestInputReport { + const input = record(value, 'input'); + exactKeys(input, ['receipts', 'undeclared', 'unused']); + validateDesktopInputReceipts(input.receipts, 'input.receipts'); + for (const [field, values] of [ + ['undeclared', desktopInputMechanisms], + ['unused', desktopRequirements], + ] as const) { + const list = input[field]; + if (list === undefined) continue; + if (!Array.isArray(list) || list.length === 0) throw new TypeError(`input.${field} must be a nonempty array.`); + for (const item of list) member(item, values); + } +} + function record(value: unknown, label: string): Record { if (!value || typeof value !== 'object' || Array.isArray(value)) throw new TypeError(`${label} must be an object.`); return value as Record; diff --git a/packages/agentic/desktop-test/src/reporting/results.test.ts b/packages/agentic/desktop-test/src/reporting/results.test.ts index 82208acd89..7953183e5d 100644 --- a/packages/agentic/desktop-test/src/reporting/results.test.ts +++ b/packages/agentic/desktop-test/src/reporting/results.test.ts @@ -11,7 +11,7 @@ import { } from './results.js'; import type { DesktopInputLaneReport, DesktopRunResult, DesktopTestResult } from './results.js'; -const accessible: DesktopInputLaneReport = { input: 'accessible', source: 'option', masking: 'host-masked', policyEnforced: false }; +const accessible: DesktopInputLaneReport = { input: 'accessible', source: 'option', masking: 'host-masked', policyEnforced: true }; function outcome( status: DesktopTestResult['status'], @@ -120,14 +120,21 @@ describe('input lane and exclusion reporting', () => { test('names the lane, its source and each exclusion on one line', () => { expect(describeDesktopSelection(result())).toBe( - 'input accessible (host-masked) from --input; excluded 2 by platform, 1 by scope, 7 by input', + 'input accessible (policy-enforced, host-masked) from --input; excluded 2 by platform, 1 by scope, 7 by input', ); - expect(describeDesktopSelection({ lane: { input: 'physical', source: 'environment', policyEnforced: false } })).toBe( - 'input physical pilot from DESKTOP_INPUT', + expect(describeDesktopSelection({ lane: { input: 'physical', source: 'environment', policyEnforced: true } })).toBe( + 'input physical pilot (policy-enforced) from DESKTOP_INPUT', ); + expect(describeDesktopSelection({ lane: { input: 'accessible', source: 'driver', policyEnforced: true } })).toBe( + 'input accessible (policy-enforced) from the driver instance', + ); + // A report written before enforcement still reads as what it was. expect( describeDesktopSelection({ lane: { input: 'accessible', source: 'driver', masking: 'verified-masked', policyEnforced: false } }), ).toBe('input accessible (verified-masked) from the driver instance'); + expect(describeDesktopSelection({ lane: { input: 'physical', source: 'environment', policyEnforced: false } })).toBe( + 'input physical pilot from DESKTOP_INPUT', + ); expect(describeDesktopSelection({})).toBeUndefined(); }); @@ -139,7 +146,7 @@ describe('input lane and exclusion reporting', () => { const xml = renderJUnit(result()); expect(xml).toContain(''); - expect(xml).toContain(''); + expect(xml).toContain(''); expect(xml).toContain(''); expect(xml).toContain(''); expect(xml).toContain(''); @@ -155,7 +162,9 @@ describe('input lane and exclusion reporting', () => { test('the Markdown summary leads with the selection and splits unsupported counts', () => { const markdown = renderRunSummary(result()); - expect(markdown).toContain('Selection: input accessible (host-masked) from --input; excluded 2 by platform, 1 by scope, 7 by input.'); + expect(markdown).toContain( + 'Selection: input accessible (policy-enforced, host-masked) from --input; excluded 2 by platform, 1 by scope, 7 by input.', + ); expect(markdown).toContain('2 unsupported (1 declared, 1 quarantined)'); expect(markdown).toContain('declared unsupported: Not executed.'); }); diff --git a/packages/agentic/desktop-test/src/reporting/results.ts b/packages/agentic/desktop-test/src/reporting/results.ts index 02fa673fe8..684a0d6ffc 100644 --- a/packages/agentic/desktop-test/src/reporting/results.ts +++ b/packages/agentic/desktop-test/src/reporting/results.ts @@ -1,6 +1,6 @@ import path from 'node:path'; -import type { EventContext, ReportFailure, SelectionExclusions, TestGap, TestStatus } from './protocol.js'; +import type { EventContext, ReportFailure, SelectionExclusions, TestGap, TestInputReport, TestStatus } from './protocol.js'; export type DesktopTestAttempt = { attempt: number; @@ -9,19 +9,24 @@ export type DesktopTestAttempt = { reason?: string; /** Present only on an `unsupported` attempt the producer declared in advance, never on a missing requirement. */ gap?: TestGap; + /** The input the attempt's session delivered, when the producer read its receipts. */ + input?: TestInputReport; failure?: ReportFailure; context?: EventContext; }; /** - * The input lane a run was asked to execute. It records the request and how the host was masked; it never claims what - * input a session delivered or enforced, hence the literal `policyEnforced: false`. + * The input lane a run was asked to execute: the request, where it came from, and how it was held. `policyEnforced` means + * every session confirmed the lane's `furn:inputMode`, so the driver refused input outside the lane; runs recorded before + * the driver enforced a policy carry `false`. `masking` names host masks that also applied. Neither qualifies a lane. */ export const desktopInputLaneSources = ['option', 'environment', 'driver'] as const; /** `driver` means the lane was read from the attached driver instance's manifest instead of being requested here. */ export type DesktopInputLaneSource = (typeof desktopInputLaneSources)[number]; +/** `host-masked`: this command started the host with the lane's masks. `verified-masked`: a pre-enforcement run checked them. */ +export type DesktopInputLaneMasking = 'host-masked' | 'verified-masked'; export type DesktopInputLaneReport = - | { input: 'accessible'; source: DesktopInputLaneSource; masking: 'host-masked' | 'verified-masked'; policyEnforced: false } - | { input: 'physical'; source: DesktopInputLaneSource; policyEnforced: false }; + | { input: 'accessible'; source: DesktopInputLaneSource; masking?: DesktopInputLaneMasking; policyEnforced: boolean } + | { input: 'physical'; source: DesktopInputLaneSource; policyEnforced: boolean }; export type DesktopSelectionReport = { excluded: SelectionExclusions }; export type DesktopTestResult = { testId: string; @@ -285,9 +290,14 @@ function describeReason(attempt: Pick): st return attempt.gap ? `${gapLabels[attempt.gap]}: ${reason}` : reason; } -/** `accessible (host-masked)`, `accessible (verified-masked)` or `physical pilot`; never a claim of enforced input. */ +/** Names the enforced input policy and any extra host mask; physical remains a pilot, not a qualification claim. */ export function describeDesktopInputLane(lane: DesktopInputLaneReport): string { - return lane.input === 'physical' ? 'physical pilot' : `accessible (${lane.masking})`; + const held = [ + ...(lane.policyEnforced ? ['policy-enforced'] : []), + ...(lane.input === 'accessible' && lane.masking ? [lane.masking] : []), + ]; + const name = lane.input === 'physical' ? 'physical pilot' : 'accessible'; + return held.length ? `${name} (${held.join(', ')})` : name; } function describeLaneSource(lane: DesktopInputLaneReport): string { diff --git a/packages/agentic/desktop-test/src/reporting/runQueries.test.ts b/packages/agentic/desktop-test/src/reporting/runQueries.test.ts index b04dad2753..6933f3b2f3 100644 --- a/packages/agentic/desktop-test/src/reporting/runQueries.test.ts +++ b/packages/agentic/desktop-test/src/reporting/runQueries.test.ts @@ -333,7 +333,8 @@ describe('persisted desktop run queries', () => { }); describe('input lane runs', () => { - const lane = { input: 'accessible', source: 'environment', masking: 'verified-masked', policyEnforced: false } as const; + const lane = { input: 'accessible', source: 'environment', policyEnforced: true } as const; + const receipts = { 'accessibility-click': 2 }; let laneReporter: DesktopRunReporter; let laneClient: DesktopRunQueryClient; @@ -348,7 +349,10 @@ describe('persisted desktop run queries', () => { service.registerRun({ runId: laneReporter.runId, root: laneReporter.root, expiresAt: Date.now() + 60_000 }); laneClient = service.connect(service.issueGrant({ runIds: [laneReporter.runId], expiresAt: Date.now() + 60_000 })); laneReporter.emit({ kind: 'selection', payload: { excluded: { platform: 1, scope: 2, input: 3 } } }); - laneReporter.emit({ kind: 'test-result', payload: { testId: 'ok', title: 'ok', attempt: 0, status: 'passed', durationMs: 1 } }); + laneReporter.emit({ + kind: 'test-result', + payload: { testId: 'ok', title: 'ok', attempt: 0, status: 'passed', durationMs: 1, input: { receipts, unused: ['keyboard'] } }, + }); laneReporter.emit({ kind: 'test-result', payload: { testId: 'gap', title: 'gap', attempt: 0, status: 'unsupported', durationMs: 0, reason: 'Declared.', gap: 'quarantine' }, @@ -377,6 +381,8 @@ describe('persisted desktop run queries', () => { { title: 'gap', status: 'unsupported', gap: 'quarantine', message: 'Declared.' }, ], }); + const persisted = JSON.parse(fs.readFileSync(path.join(laneReporter.root, 'run.json'), 'utf8')) as DesktopRunResult; + expect(persisted.tests[0].attempts[0].input).toEqual({ receipts, unused: ['keyboard'] }); }); test('a declared gap edited out of the retained report no longer reads as a passing lane', async () => { @@ -405,8 +411,23 @@ describe('persisted desktop run queries', () => { }); test.each([ - ['a lane that claims enforcement', (result: DesktopRunResult) => Object.assign(result, { lane: { ...lane, policyEnforced: true } })], - ['a physical lane with masking', (result: DesktopRunResult) => Object.assign(result, { lane: { ...lane, input: 'physical' } })], + [ + 'an accessible lane with neither masks nor enforcement', + (result: DesktopRunResult) => Object.assign(result, { lane: { ...lane, policyEnforced: false } }), + ], + [ + 'a physical lane with masking', + (result: DesktopRunResult) => Object.assign(result, { lane: { ...lane, input: 'physical', masking: 'host-masked' } }), + ], + [ + 'undeclared input on a passed attempt', + (result: DesktopRunResult) => + Object.assign(result.tests[0].attempts[0], { input: { receipts, undeclared: ['accessibility-click'] } }), + ], + [ + 'an unknown input field', + (result: DesktopRunResult) => Object.assign(result.tests[0].attempts[0], { input: { receipts, extra: 1 } }), + ], [ 'a negative exclusion count', (result: DesktopRunResult) => Object.assign(result, { selection: { excluded: { platform: -1, scope: 0, input: 0 } } }), diff --git a/packages/agentic/desktop-test/src/reporting/runQueries.ts b/packages/agentic/desktop-test/src/reporting/runQueries.ts index 1f6843a2be..12a5f8d679 100644 --- a/packages/agentic/desktop-test/src/reporting/runQueries.ts +++ b/packages/agentic/desktop-test/src/reporting/runQueries.ts @@ -4,8 +4,15 @@ import { createHash, randomUUID } from 'node:crypto'; import { isNativeError } from 'node:util/types'; import { desktopArtifactPath } from './collector.js'; -import { DesktopReportedError, reportStatuses, redactDesktopSecrets, selectionExclusionReasons, testGaps } from './protocol.js'; -import type { SelectionExclusions, TestGap, TestStatus } from './protocol.js'; +import { + DesktopReportedError, + reportStatuses, + redactDesktopSecrets, + selectionExclusionReasons, + testGaps, + validateTestInputReport, +} from './protocol.js'; +import type { SelectionExclusions, TestGap, TestInputReport, TestStatus } from './protocol.js'; import { desktopInputLaneSources, summarizeDesktopRun } from './results.js'; import type { DesktopInputLaneReport, DesktopRunResult, DesktopSelectionReport } from './results.js'; @@ -363,12 +370,19 @@ function parseResult(text: string, runId: string): DesktopRunResult & { terminal const gap = attempt.gap; if (gap !== undefined && (!testGaps.includes(gap as TestGap) || attempt.status !== 'unsupported')) throw new TypeError('Invalid persisted desktop test gap.'); + let input: TestInputReport | undefined; + if (attempt.input !== undefined) { + validateTestInputReport(attempt.input); + if (attempt.input.undeclared && attempt.status !== 'failed') throw new TypeError('Invalid persisted desktop test input.'); + input = attempt.input; + } return { attempt: attempt.attempt, durationMs: attempt.durationMs, status: attempt.status, ...(typeof reason === 'string' ? { reason } : {}), ...(gap !== undefined ? { gap: gap as TestGap } : {}), + ...(input !== undefined ? { input } : {}), ...(attempt.failure !== undefined ? { failure: new DesktopReportedError(attempt.failure).failure } : {}), }; }); @@ -416,10 +430,13 @@ function laneField(value: unknown): DesktopInputLaneReport | undefined { if (value === undefined) return undefined; const lane = record(value); const source = desktopInputLaneSources.find((candidate) => candidate === lane.source); - if (lane.policyEnforced !== false || source === undefined) throw new TypeError('Invalid persisted desktop input lane.'); - if (lane.input === 'physical' && lane.masking === undefined) return { input: 'physical', source, policyEnforced: false }; + const policyEnforced = lane.policyEnforced; + if (typeof policyEnforced !== 'boolean' || source === undefined) throw new TypeError('Invalid persisted desktop input lane.'); + if (lane.input === 'physical' && lane.masking === undefined) return { input: 'physical', source, policyEnforced }; + // A run recorded before the driver enforced a policy always named its masking, because the masks alone held its lane. + if (lane.input === 'accessible' && lane.masking === undefined && policyEnforced) return { input: 'accessible', source, policyEnforced }; if (lane.input === 'accessible' && (lane.masking === 'host-masked' || lane.masking === 'verified-masked')) - return { input: 'accessible', source, masking: lane.masking, policyEnforced: false }; + return { input: 'accessible', source, masking: lane.masking, policyEnforced }; throw new TypeError('Invalid persisted desktop input lane.'); } diff --git a/packages/agentic/desktop-test/src/wdio/configuration.ts b/packages/agentic/desktop-test/src/wdio/configuration.ts index f16768c542..b52c13597a 100644 --- a/packages/agentic/desktop-test/src/wdio/configuration.ts +++ b/packages/agentic/desktop-test/src/wdio/configuration.ts @@ -95,6 +95,8 @@ export async function createDesktopWdioConfiguration(settings: DesktopWdioSettin 'furn:target': settings.targetId, 'furn:launchMode': settings.ownership === 'external' ? 'attach' : 'launch', 'furn:clickMode': resolveDesktopClickMode(undefined, 'auto', selection?.lane), + // A lane's session refuses input its lane never delivers; without one, every mechanism stays available. + 'furn:inputMode': selection?.lane ?? 'compatibility', }, ], beforeSession: async (...args: unknown[]) => { @@ -105,7 +107,12 @@ export async function createDesktopWdioConfiguration(settings: DesktopWdioSettin installDesktopWebdriverCommands(browser, settings.url); // Before the application is touched: a lane that cannot vouch for its tests must stop here. if (settings.framework === 'jasmine') { - selectDesktopSpecs(desktopJasmineRegistry(), currentDesktopJasmineEnvironment(), browser.capabilities['furn:features']); + selectDesktopSpecs( + desktopJasmineRegistry(), + currentDesktopJasmineEnvironment(), + browser.capabilities['furn:features'], + browser.capabilities['furn:inputMode'], + ); } if ((await browser.getWindowHandles()).length === 0) throw new Error('Native application did not expose a window.'); if (readiness?.testId) { diff --git a/packages/agentic/desktop-test/src/wdio/reporter.ts b/packages/agentic/desktop-test/src/wdio/reporter.ts index 7379061b54..bef914a414 100644 --- a/packages/agentic/desktop-test/src/wdio/reporter.ts +++ b/packages/agentic/desktop-test/src/wdio/reporter.ts @@ -116,6 +116,8 @@ export default class DesktopWdioReporter extends WDIOReporter { const gap = notExecuted && entry?.kind === 'gap' ? entry.gap : undefined; const missing = notExecuted && entry?.kind === 'run' ? entry.missing : undefined; const status = gap || missing?.length ? 'unsupported' : test.state === 'pending' ? 'skipped' : test.state; + // A body that called pending() also delivered input, but only an executed test is judged by it. + const input = entry?.kind === 'run' && (status === 'passed' || status === 'failed') ? entry.input : undefined; this.report( { kind: 'test-result', @@ -127,6 +129,7 @@ export default class DesktopWdioReporter extends WDIOReporter { durationMs: test.duration, ...(gap ? { gap: gap.kind, reason: gap.reason } : {}), ...(missing?.length ? { reason: `Missing desktop requirements: ${missing.join(', ')}.` } : {}), + ...(input ? { input } : {}), ...(status === 'skipped' ? { reason: test.pendingReason || 'Skipped by the test framework.' } : {}), ...(status === 'failed' ? { failure: this.failure(`${key}/failure`, test.error ?? new AggregateError(test.errors ?? [], 'WDIO test failed.'), 'test') } diff --git a/packages/agentic/storybook-desktop/config/transform-story-tests.cjs b/packages/agentic/storybook-desktop/config/transform-story-tests.cjs index ae64781038..6952c743b1 100644 --- a/packages/agentic/storybook-desktop/config/transform-story-tests.cjs +++ b/packages/agentic/storybook-desktop/config/transform-story-tests.cjs @@ -1,8 +1,9 @@ +const fs = require('node:fs'); const { isBuiltin, createRequire } = require('node:module'); const path = require('node:path'); const { pathToFileURL } = require('node:url'); -const { transformFromAstSync, traverse, types } = require('@babel/core'); +const { parseSync, transformFromAstSync, traverse, types } = require('@babel/core'); const transformTypescript = require('@babel/plugin-transform-typescript'); const nodeGlobals = new Set( @@ -336,28 +337,184 @@ function directInputCall(call) { return undefined; } -/** - * Rejects a case that also runs in the accessible input lane but sends physical input itself: the keyboard commands, the value - * commands that may type, and WebDriver actions with key, pointer or wheel sources. The check reads literals only. Input sent - * through a helper or an alias is stopped by the accessible lane's host masks when the case runs. - */ -function rejectDirectInput(callback, { name, input, inputSource }) { - if (input === 'physical') return; +/** The string a literal `import()`, awaited or not, loads; undefined for anything else. */ +function dynamicImportSource(expression) { + expression = unwrap(expression); + if (expression?.isAwaitExpression()) expression = unwrap(expression.get('argument')); + if (expression?.isImportExpression?.()) { + const source = expression.get('source'); + return source.isStringLiteral() ? source.node.value : undefined; + } + if (!expression?.isCallExpression() || !types.isImport(expression.node.callee) || expression.node.arguments.length !== 1) + return undefined; + const source = expression.get('arguments.0'); + return source.isStringLiteral() ? source.node.value : undefined; +} + +const helperExtensions = /\.(?:[cm]?[jt]s)$/; + +/** A relative helper module this check can read, or undefined for a package, a builtin or a file it cannot find. */ +function resolveHelper(specifier, fromFile) { + if (!specifier.startsWith('.')) return undefined; + try { + const resolved = createRequire(path.resolve(fromFile)).resolve(specifier); + return helperExtensions.test(resolved) ? resolved : undefined; + } catch { + return undefined; + } +} + +/** The module and export a callee names when it is a helper loaded from a relative module, by static or literal dynamic import. */ +function importedHelper(callee, fromFile) { + callee = unwrap(callee); + if (callee.isIdentifier()) { + const binding = callee.scope.getBinding(callee.node.name); + if (binding?.path.isImportSpecifier()) { + const imported = binding.path.node.imported; + const file = resolveHelper(binding.path.parentPath.node.source.value, fromFile); + return file && { file, name: types.isIdentifier(imported) ? imported.name : imported.value }; + } + // const { helper } = await import('./helpers.ts'); + if (!binding?.path.isVariableDeclarator() || !binding.path.get('id').isObjectPattern()) return undefined; + const source = dynamicImportSource(binding.path.get('init')); + const file = source && resolveHelper(source, fromFile); + if (!file) return undefined; + for (const property of binding.path.get('id.properties')) { + if (!property.isObjectProperty() || property.node.computed) continue; + if (types.isIdentifier(property.node.value, { name: callee.node.name })) return { file, name: keyName(property) }; + } + return undefined; + } + if (!callee.isMemberExpression() && !callee.isOptionalMemberExpression()) return undefined; + const name = memberName(callee); + const object = unwrap(callee.get('object')); + // (await import('./helpers.ts')).helper(), helpers.helper() after `const helpers = await import(...)`, or a namespace import. + let source = dynamicImportSource(object); + if (source === undefined && object.isIdentifier()) { + const binding = object.scope.getBinding(object.node.name); + if (binding?.path.isVariableDeclarator() && binding.path.get('id').isIdentifier()) + source = dynamicImportSource(binding.path.get('init')); + else if (binding?.path.isImportNamespaceSpecifier()) source = binding.path.parentPath.node.source.value; + } + const file = name && source !== undefined ? resolveHelper(source, fromFile) : undefined; + return file && { file, name }; +} + +/** Reads a helper module once per check: its top-level functions, and the exports that name them. */ +function readHelperModule(file, context) { + if (context.modules.has(file)) return context.modules.get(file); + let helper = null; + try { + const ast = parseSync(fs.readFileSync(file, 'utf8'), { + babelrc: false, + configFile: false, + filename: file, + sourceType: 'module', + parserOpts: { plugins: ['typescript'] }, + }); + let program; + traverse(ast, { + Program(found) { + program = found; + found.stop(); + }, + }); + helper = { file, program, functions: new Map(), exports: new Map() }; + for (const statement of program.get('body')) { + const exported = statement.isExportNamedDeclaration(); + const declaration = exported ? statement.get('declaration') : statement; + if (declaration.isFunctionDeclaration() && declaration.node.id) { + helper.functions.set(declaration.node.id.name, declaration); + if (exported) helper.exports.set(declaration.node.id.name, declaration.node.id.name); + } else if (declaration.isVariableDeclaration()) { + for (const declarator of declaration.get('declarations')) { + const init = declarator.node.init ? unwrap(declarator.get('init')) : undefined; + if (!declarator.get('id').isIdentifier() || !(init?.isFunctionExpression() || init?.isArrowFunctionExpression())) continue; + helper.functions.set(declarator.node.id.name, init); + if (exported) helper.exports.set(declarator.node.id.name, declarator.node.id.name); + } + } + if (exported && !statement.node.source) { + for (const specifier of statement.get('specifiers')) { + if (!specifier.isExportSpecifier()) continue; + const exportedName = types.isIdentifier(specifier.node.exported) ? specifier.node.exported.name : specifier.node.exported.value; + helper.exports.set(exportedName, specifier.node.local.name); + } + } + } + } catch { + // A helper this check cannot parse is left to the run-time policy, like any other input it cannot see. + helper = null; + } + context.modules.set(file, helper); + return helper; +} + +/** The first physical input a helper function sends, directly or through the helpers it calls. Cycles end at the first visit. */ +function helperFunctionInput(helper, name, context) { + const key = `${helper.file}#${name}`; + if (!helper.functions.has(name)) return undefined; + if (context.functions.has(key)) return context.functions.get(key); + context.functions.set(key, undefined); + const found = findPhysicalInput(helper.functions.get(name), helper.file, context, helper); + context.functions.set(key, found); + return found; +} + +/** A call to a helper that sends physical input, as the chain of helpers that leads to the call that sends it. */ +function helperInputCall(call, fromFile, context, helper) { + const callee = unwrap(call.get('callee')); + let target = importedHelper(callee, fromFile); + let local; + if (target) { + const imported = readHelperModule(target.file, context); + local = imported && { helper: imported, name: imported.exports.get(target.name) }; + } else if (helper && callee.isIdentifier() && helper.functions.has(callee.node.name)) { + // Another function in the same helper module, unless a local binding shadows it. + if (callee.scope.getBinding(callee.node.name)?.scope !== helper.program.scope) return undefined; + target = { name: callee.node.name }; + local = { helper, name: callee.node.name }; + } + if (!local?.name) return undefined; + const found = helperFunctionInput(local.helper, local.name, context); + return found && { call: found.call, requires: found.requires, via: [target.name, ...(found.via ?? [])] }; +} + +/** The first call under `root` that sends physical input, directly or through a relative helper module. */ +function findPhysicalInput(root, file, context, helper) { let finding; - callback.traverse({ + root.traverse({ 'CallExpression|OptionalCallExpression'(call) { - const found = finding ? undefined : directInputCall(call); - if (found) finding = { ...found, path: call }; + if (finding) return; + const found = directInputCall(call) ?? (file ? helperInputCall(call, file, context, helper) : undefined); + if (found) { + finding = { ...found, path: call }; + call.stop(); + } }, }); + return finding; +} + +/** + * Rejects a case that also runs in the accessible input lane but sends physical input: the keyboard commands, the value + * commands that may type, and WebDriver actions with key, pointer or wheel sources, called directly or through a helper + * imported from a relative module. The check reads literals only. Input sent through an alias or a package helper is + * refused by the accessible lane's input policy when the case runs, and the case's receipts are checked against its + * declaration in every lane. + */ +function rejectDirectInput(callback, { name, input, inputSource, sourceFile }) { + if (input === 'physical') return; + const finding = findPhysicalInput(callback, sourceFile, { modules: new Map(), functions: new Map() }); if (!finding) return; const subject = name ? `"${name}"` : 'This case'; const declared = inputSource === 'declared'; + const caller = finding.via ? `helper ${finding.via.join(' > ')} calls ${finding.call}` : `calls ${finding.call}`; if (finding.requires === undefined) { const declaration = declared ? "input: 'physical'" : 'requires for the physical input it sends (keyboard, physical-click or wheel)'; fail( finding.path, - `${subject} runs in the accessible input lane, but calls ${finding.call} with an argument this check cannot read. ` + + `${subject} runs in the accessible input lane, but ${caller} with an argument this check cannot read. ` + `Pass a literal list of input sources, or declare ${declaration}.`, ); } @@ -365,9 +522,12 @@ function rejectDirectInput(callback, { name, input, inputSource }) { const declaration = declared ? "input: 'physical'" : `requires: [${finding.requires.map((requirement) => `'${requirement}'`).join(', ')}]`; + const sender = finding.via + ? `helper ${finding.via.join(' > ')} sends physical ${kinds} input with ${finding.call}` + : `${finding.call} sends physical ${kinds} input`; fail( finding.path, - `${subject} runs in the accessible input lane, but ${finding.call} sends physical ${kinds} input. ` + + `${subject} runs in the accessible input lane, but ${sender}. ` + `Declare ${declaration} so it runs only in the physical lane, or remove the call.`, ); } diff --git a/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts b/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts index e620c955b8..d9d464e526 100644 --- a/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts +++ b/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts @@ -18,7 +18,13 @@ declare namespace storyTests { function compileWdioTest(callback: Callback['callback'], sourceFile: string, source: string): string; function rejectDirectInput( callback: Callback['callback'], - descriptor: { name?: string; input: 'accessible' | 'physical' | 'both'; inputSource: 'declared' | 'derived' }, + descriptor: { + name?: string; + input: 'accessible' | 'physical' | 'both'; + inputSource: 'declared' | 'derived'; + /** The story file, so helpers imported from relative modules are followed. */ + sourceFile?: string; + }, ): void; } diff --git a/packages/agentic/storybook-desktop/references/commands.md b/packages/agentic/storybook-desktop/references/commands.md index a82fa228d0..34ea5326e9 100644 --- a/packages/agentic/storybook-desktop/references/commands.md +++ b/packages/agentic/storybook-desktop/references/commands.md @@ -69,9 +69,10 @@ resolved input, target platform, and scope. Without a flag the lane comes from `DESKTOP_INPUT`, then from the driver instance a `test` attaches to; none of them means compatibility mode, which runs every case as before. `driver --input` starts the driver for that lane and records it for subsequent tests against the -instance. An `accessible` driver masks keyboard, physical click, and wheel, and -`test --input accessible` verifies the mask before running anything: a driver -that is not masked fails the run rather than letting a case move the pointer. +instance. Each test session confirms its strict input policy before running; +the accessible policy rejects keyboard, pointer, and wheel input before +dispatch. An accessible driver also masks those host features as defense in +depth, but a test can attach to an unmasked driver. Legacy callback collections are not eligible for input-lane runs. Input lanes cannot be combined with an explicit non-`auto` click mode. See [story tests](story-tests.md#run-an-input-lane). diff --git a/packages/agentic/storybook-desktop/references/story-tests.md b/packages/agentic/storybook-desktop/references/story-tests.md index 438c0e09a7..2f4b5a3025 100644 --- a/packages/agentic/storybook-desktop/references/story-tests.md +++ b/packages/agentic/storybook-desktop/references/story-tests.md @@ -180,15 +180,19 @@ non-default `--click-mode`. `test --list` shows, for each story, the cases the lane runs with their resolved input, requirements, platforms, and scope, and the cases it excludes with the reason; add `--input` to preview a lane. -The accessible lane holds a case to accessible input by masking the host, not by -trusting the case: the driver reports keyboard, physical click, and wheel as -unavailable, so a case that tries them fails instead of silently moving the -pointer. `smoke` and `driver --input accessible` start the driver already -masked. `test --input accessible` attaches to a running driver and verifies the -mask from the session's reported features before it runs anything. A driver that -does not report all three as unavailable stops the run as an infrastructure -failure rather than running unmasked, so attach the lane to a driver started -with `driver --input accessible`. +Each case session confirms its `furn:inputMode` before the callback runs. The +accessible mode refuses keyboard, pointer, and wheel input before dispatch; the +physical mode pins clicks to the pointer while allowing accessibility actions. +`smoke` and `driver --input accessible` also mask the host as defense in depth, +but `test --input accessible` can safely attach to an unmasked driver because +the session policy is enforced independently. Before teardown, the runner reads +the session's delivery receipts and fails a case if it delivered a mechanism +not covered by its `requires`; unused requirements are reported, not failed. + +In an accessible session, `browser.$(...).setValue(...)` is rejected before +WDIO's client-side clear step, which could otherwise change the field before +keyboard input is refused. `clearValue()` remains available and uses a semantic +clear. A lane that executes no case is not a passing run, and neither is a lane run that leaves a case unsupported or skipped without a declared gap. Excluded cases diff --git a/packages/agentic/storybook-desktop/src/cli/inputLane.test.ts b/packages/agentic/storybook-desktop/src/cli/inputLane.test.ts index 1d9202f145..dd473bf24e 100644 --- a/packages/agentic/storybook-desktop/src/cli/inputLane.test.ts +++ b/packages/agentic/storybook-desktop/src/cli/inputLane.test.ts @@ -37,23 +37,22 @@ describe('command input lane', () => { expect(driverInput).not.toHaveBeenCalled(); }); - test('labels how an accessible lane is masked by who owns the driver, and never a physical lane', () => { + test('enforces every lane by policy, and records host masks only where smoke started the driver', () => { expect(resolveCommandInputLane({ command: 'smoke', runsTests: true, flag: 'accessible' })).toEqual({ input: 'accessible', source: 'option', masking: 'host-masked', - policyEnforced: false, + policyEnforced: true, }); expect(resolveCommandInputLane({ command: 'test', runsTests: true, flag: 'accessible' })).toEqual({ input: 'accessible', source: 'option', - masking: 'verified-masked', - policyEnforced: false, + policyEnforced: true, }); expect(resolveCommandInputLane({ command: 'smoke', runsTests: true, flag: 'physical' })).toEqual({ input: 'physical', source: 'option', - policyEnforced: false, + policyEnforced: true, }); }); diff --git a/packages/agentic/storybook-desktop/src/cli/inputLane.ts b/packages/agentic/storybook-desktop/src/cli/inputLane.ts index 8a6e36dd39..7665c5a512 100644 --- a/packages/agentic/storybook-desktop/src/cli/inputLane.ts +++ b/packages/agentic/storybook-desktop/src/cli/inputLane.ts @@ -20,12 +20,12 @@ export type CommandInputLaneRequest = { /** * The lane a `test` or `smoke` run holds its tests to, and where it came from: `--input`, then `DESKTOP_INPUT`, then * the driver instance `test` attaches to. No lane is compatibility mode, which keeps the behavior from before lanes. - * An accessible lane's masking depends on who owns the driver: `smoke` starts it masked, while `test` attaches to one - * and verifies the mask before it runs anything. + * Every session confirms the lane's input policy. `smoke` also starts its driver masked for an accessible lane, while + * `test` attaches to whatever driver is running and relies on the policy alone. */ export function resolveCommandInputLane(request: CommandInputLaneRequest): DesktopInputLaneReport | undefined { if (!request.runsTests) return undefined; - const masking = request.command === 'smoke' ? 'host-masked' : 'verified-masked'; + const masking = request.command === 'smoke' ? 'host-masked' : undefined; const requested = requestedDesktopInputLane({ flag: request.flag, environment: request.environment }); if (requested) return createDesktopInputLane(requested.input, requested.source, masking); const input = request.driverInput?.(); diff --git a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts index 16766657bb..0d97f22c38 100644 --- a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts +++ b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts @@ -1,5 +1,7 @@ import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; import { createRequire } from 'node:module'; +import os from 'node:os'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; import vm from 'node:vm'; @@ -390,6 +392,101 @@ describe('extractWdioTests', () => { expect(() => extractWdioTests(desktopCase(body), sourceFile)).not.toThrow(); }); + describe('through relative helpers', () => { + let root: string; + let story: string; + + beforeAll(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-wdio-helpers-')); + story = path.join(root, 'components', 'example.stories.tsx'); + fs.mkdirSync(path.join(root, 'common')); + fs.mkdirSync(path.join(root, 'components')); + fs.writeFileSync( + path.join(root, 'common', 'focus.wdio.ts'), + `import type { Browser } from 'webdriverio'; + export async function expectFocused(browser: Browser, id: string): Promise { await browser.$(id); } + export async function focusByTab(browser: Browser, id: string): Promise { + await expectFocused(browser, id); + await browser.keys('\\uE004'); + } + export async function tabTwice(browser: Browser): Promise { await focusByTab(browser, 'a'); } + export async function perform(browser: Browser, actions: never[]): Promise { await browser.performActions(actions); } + export const clickPhysical = async (browser: Browser) => { + await browser.performActions([${pointer}]); + }; + async function loop(browser: Browser): Promise { await again(browser); } + async function again(browser: Browser): Promise { await loop(browser); } + export { loop as cycle }; + export const label = 'not a function';`, + ); + fs.writeFileSync( + path.join(root, 'components', 'menu.wdio.ts'), + `export async function openFromKeyboard(context: { browser: unknown }): Promise { + const { focusByTab } = await import('../common/focus.wdio.ts'); + await focusByTab(context.browser as never, 'entry'); + } + export { expectFocused } from '../common/focus.wdio.ts';`, + ); + }); + + afterAll(() => fs.rmSync(root, { recursive: true, force: true })); + + test.each([ + [ + 'a destructured helper', + `const { focusByTab } = await import('../common/focus.wdio.ts'); await focusByTab(browser, 'entry');`, + /helper focusByTab sends physical keyboard input with browser\.keys.*requires: \['keyboard'\]/, + ], + [ + 'a helper called on the awaited module', + `await (await import('../common/focus.wdio.ts')).clickPhysical(browser);`, + /helper clickPhysical sends physical pointer input with performActions.*requires: \['physical-click'\]/, + ], + [ + 'a helper called on a module binding', + `const focus = await import('../common/focus.wdio.ts'); await focus.focusByTab(browser, 'entry');`, + /helper focusByTab sends physical keyboard input/, + ], + [ + 'a helper that calls another helper in its module', + `const { tabTwice } = await import('../common/focus.wdio.ts'); await tabTwice(browser);`, + /helper tabTwice > focusByTab sends physical keyboard input/, + ], + [ + 'a helper whose actions it cannot read', + `const { perform } = await import('../common/focus.wdio.ts'); await perform(browser, []);`, + /helper perform calls performActions with an argument this check cannot read/, + ], + [ + 'a helper that imports another helper', + `const { openFromKeyboard } = await import('./menu.wdio.ts'); await openFromKeyboard(context);`, + /helper openFromKeyboard > focusByTab sends physical keyboard input with browser\.keys/, + ], + ])('rejects %s that sends physical input', (_name, body, message) => { + expect(() => extractWdioTests(desktopCase(body), story)).toThrow(message); + }); + + test.each([ + [ + 'a helper that sends no input', + `const { expectFocused } = await import('../common/focus.wdio.ts'); await expectFocused(browser, 'a');`, + ], + ['helpers that call each other in a cycle', `const { cycle } = await import('../common/focus.wdio.ts'); await cycle(browser);`], + [ + 'a re-exported helper, which the check does not follow', + `const { expectFocused } = await import('./menu.wdio.ts'); await expectFocused(browser, 'a');`, + ], + ['a package helper', `const { join } = await import('node:path'); join('a');`], + ])('allows %s', (_name, body) => { + expect(() => extractWdioTests(desktopCase(body), story)).not.toThrow(); + }); + + test('allows a helper that sends input in a case that declares it', () => { + const body = `const { focusByTab } = await import('../common/focus.wdio.ts'); await focusByTab(browser, 'entry');`; + expect(() => extractWdioTests(desktopCase(body, `{ requires: ['keyboard'] }`), story)).not.toThrow(); + }); + }); + test('does not read legacy callbacks, which declare no input', () => { expect(() => extractWdioTests(`export const Example = { wdio: { legacy: async ({ browser }) => { await browser.keys('a'); } } };`, sourceFile), diff --git a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts index 4bf2066810..eb18f658fd 100644 --- a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts +++ b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts @@ -30,7 +30,7 @@ export function extractWdioTests(code: string, sourceFile: string): ReadonlyMap< const compiledCode = compileWdioTest(callback, sourceFile, code); const descriptor = options === undefined ? undefined : validateDesktopCaseOptions(options, `${sourceFile}.wdio`); if (descriptor !== undefined) { - rejectDirectInput(callback, { name, input: descriptor.input, inputSource: descriptor.inputSource }); + rejectDirectInput(callback, { name, input: descriptor.input, inputSource: descriptor.inputSource, sourceFile }); } return { ...(name === undefined ? {} : { name }), diff --git a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts index 524bd88ae3..87c5941294 100644 --- a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts +++ b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts @@ -5,14 +5,15 @@ import { pathToFileURL } from 'node:url'; import type { DesktopStoryManifest } from '@fluentui-react-native/desktop-driver'; import { + compareDesktopInputReceipts, + describeUndeclaredDesktopInput, missingDesktopRequirements, resolveDesktopClickMode, - unmaskedAccessibleFeatures, type DesktopArtifact, } from '@fluentui-react-native/desktop-driver/authoring'; import { ArtifactManager, captureDesktopFailureEvidence } from '@fluentui-react-native/desktop-driver/artifacts'; import { connectDesktopWebdriver } from '@fluentui-react-native/desktop-driver/wdio'; -import type { DesktopReportWriter } from '@fluentui-react-native/desktop-test/reporting'; +import type { DesktopReportWriter, TestInputReport } from '@fluentui-react-native/desktop-test/reporting'; import { DesktopReportedError } from '@fluentui-react-native/desktop-test/reporting'; import type { DesktopStorybookConfig } from '../config/makeDesktopStorybookConfig.js'; @@ -40,6 +41,8 @@ export type WdioStoryTestResult = { error?: string; skipReason?: string; gap?: 'unsupported' | 'quarantine'; + /** The input the case's session delivered, judged against its declaration when it has one. */ + input?: TestInputReport; evidenceErrors?: string[]; artifacts?: readonly DesktopArtifact[]; }; @@ -78,15 +81,34 @@ class MissingRequirementsError extends Error { } } -/** The driver cannot show it disabled physical input, so an accessible lane could not keep its guarantee. */ -class UnmaskedLaneError extends Error { - constructor(unmasked: readonly string[]) { +/** The session did not confirm the input policy it was created with, so no case it runs can be held to that policy. */ +class InputPolicyError extends Error { + constructor(requested: string, confirmed: unknown) { super( - `The accessible input lane needs a driver that reports ${unmasked.join(', ')} disabled, and this one does not. Start it with "driver --input accessible".`, + `The driver did not confirm the ${requested} input policy (it reported ${typeof confirmed === 'string' ? confirmed : 'none'}). Restart "storybook driver" from this version of storybook-desktop.`, ); } } +/** The case delivered input its declaration does not account for. */ +class UndeclaredInputError extends Error {} + +/** + * Reads what the case's session delivered and judges it against the declaration; a legacy callback declares nothing to + * judge. Only a case that ran to completion reports declarations it did not use, since one that stopped early could not. + */ +async function readDeliveredInput( + desktop: Awaited>, + descriptor: Parameters[0], + clickMode: ReturnType, + completed: boolean, +): Promise { + const { receipts } = await desktop.input(); + if (!descriptor) return { receipts }; + const { undeclared, unused } = compareDesktopInputReceipts(descriptor, receipts, clickMode); + return { receipts, ...(undeclared.length ? { undeclared } : {}), ...(completed && unused.length ? { unused } : {}) }; +} + export async function runWdioStoryTests( options: RunWdioStoryTestsOptions, runner?: DesktopCommandRunner, @@ -194,9 +216,11 @@ export async function runWdioStoryTests( continue; } const clickMode = resolveDesktopClickMode(descriptor, settings.clickMode, settings.input); + const inputMode = settings.input ?? 'compatibility'; // The supervisor owns the session, so even a crashed or timed-out Node worker cannot strand it. const desktop = await connectDesktopWebdriver({ clickMode, + inputMode, launchMode: 'attach', platformName: options.platform === 'macos' ? 'macos' : 'windows', targetId: options.targetId, @@ -209,12 +233,11 @@ export async function runWdioStoryTests( if (live.endpoint !== options.platform || live.platformManifestDigest !== options.manifest.platformManifestDigest) { throw new Error('The running Storybook manifest is stale or targets a different platform. Restart storybook driver and the app.'); } - if (settings.input === 'accessible') { - // A driver that cannot report its physical input disabled fails closed rather than silently running unmasked. - phase = 'lane verification'; - const unmasked = unmaskedAccessibleFeatures(desktop.browser.capabilities['furn:features']); - if (unmasked.length) throw new UnmaskedLaneError(unmasked); - } + // The driver refuses input outside the policy before dispatch, so a lane holds even on a host with no masks. A + // driver that does not confirm the policy fails closed rather than running cases it cannot hold or attribute. + phase = 'input policy verification'; + const confirmed = desktop.browser.capabilities['furn:inputMode']; + if (confirmed !== inputMode) throw new InputPolicyError(inputMode, confirmed); const missing = missingDesktopRequirements(descriptor, desktop.browser.capabilities['furn:features'], clickMode); if (missing.length) throw new MissingRequirementsError(missing); phase = 'navigation'; @@ -268,12 +291,23 @@ export async function runWdioStoryTests( if (outcome.status === 'skipped') { result.skipReason = outcome.skipReason; } + // Read before teardown: the receipts belong to the session and end with it. + phase = 'input verification'; + result.input = await readDeliveredInput(desktop, descriptor, clickMode, outcome.status === 'passed'); + if (result.input.undeclared) throw new UndeclaredInputError(describeUndeclaredDesktopInput(result.input.undeclared)); } catch (error) { if (error instanceof MissingRequirementsError) { result.status = 'unsupported'; result.skipReason = error.message; - } else if (error instanceof UnmaskedLaneError) { + } else if (error instanceof InputPolicyError) { failure = error; + } else if (error instanceof UndeclaredInputError) { + // The case ran as written, so there is no application state to capture; the declaration is what is wrong. + failure = error; + result.status = 'failed'; + executionFailures.push(error); + result.error = error.message; + writeDesktopStorybookFailure(`wdio "${displayName}" during ${phase}`, error, options.errorOutput); } else { failure = error; executionFailures.push(error); @@ -297,6 +331,8 @@ export async function runWdioStoryTests( options.report?.fail(new Error(evidenceError), 'evidence', testId); writeDesktopStorybookFailure(`evidence for "${displayName}"`, evidenceError, options.errorOutput); } + // What a failed case delivered attributes the failure. A session that cannot answer leaves it unknown. + if (phase === 'execution') result.input = await readDeliveredInput(desktop, descriptor, clickMode, false).catch(() => undefined); } } phase = 'session cleanup'; @@ -310,8 +346,8 @@ export async function runWdioStoryTests( ); } // Every case would fail the same way, so stop here and let the run report it once, against this case. - if (failure instanceof UnmaskedLaneError) { - phase = 'lane verification'; + if (failure instanceof InputPolicyError) { + phase = 'input policy verification'; throw failure; } result.durationMs = Date.now() - started; @@ -326,6 +362,7 @@ export async function runWdioStoryTests( status: result.status, durationMs: result.durationMs, ...(['skipped', 'unsupported'].includes(result.status) ? { reason: result.skipReason } : {}), + ...(result.input ? { input: result.input } : {}), ...(failure !== undefined ? { failure: options.report.failure(failure, 'test') } : {}), }, }, diff --git a/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs b/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs index 3736a9666e..44dd1bbb3c 100644 --- a/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs +++ b/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs @@ -1,5 +1,7 @@ +const assert = require('node:assert/strict'); const { spawn } = require('node:child_process'); const fs = require('node:fs'); +const http = require('node:http'); const os = require('node:os'); const path = require('node:path'); const { pathToFileURL } = require('node:url'); @@ -100,6 +102,22 @@ async function main() { await expect(await browser.$('~button-primary')).toBeEnabled(); }), ]}; + export const Receipts = { wdio: [ + desktopIt('declared click', { requires: ['click'] }, async ({ browser }) => { + await (await browser.$('~button-primary')).click(); + }), + desktopIt('undeclared click', async ({ browser }) => { + await (await browser.$('~button-primary')).click(); + }), + desktopIt('unused keyboard', { requires: ['keyboard'] }, async ({ browser, expect }) => { + await expect(await browser.$('~button-primary')).toBeEnabled(); + }), + // An alias hides the keyboard from the authoring check, so only the run-time policy and receipts can catch it. + desktopIt('typing', { requires: ['click'] }, async ({ browser }) => { + const driver = browser; + await driver.keys(['Tab']); + }), + ]}; `, ); const storyPackage = { name: 'inline-fixture', root, manifest: { name: 'inline-fixture' }, storyPatterns: ['*.stories.ts'] }; @@ -339,22 +357,70 @@ async function main() { await restrictedButtonHarness.close(); } const laneResults = (results) => results.map(({ testName, status }) => ({ testName, status })); - // The accessible lane trusts the driver's own report, so a driver that does not show physical input disabled fails closed. - const unmaskedReport = new DesktopRunReporter({ + // The driver enforces the lane's policy itself, so an accessible lane holds on a driver with physical input enabled. + const unmaskedLane = laneResults(await run('lane', { input: 'accessible' })); + // A driver that does not confirm the policy fails closed: it could neither hold the lane nor attribute its input. + const unconfirmedProxy = await createUnconfirmingProxy(harness.server.url); + const unconfirmedReport = new DesktopRunReporter({ artifactsRoot: path.join(root, 'lane-reports'), - lane: { input: 'accessible', source: 'option', masking: 'host-masked', policyEnforced: false }, + lane: { input: 'accessible', source: 'option', policyEnforced: true }, summary: false, }); - let unmaskedLane; + let unconfirmedLane; try { - await run('lane', { input: 'accessible', report: unmaskedReport }); - throw new Error('An accessible lane must not run against a driver that reports physical input enabled.'); + // eslint-disable-next-line @microsoft/sdl/no-insecure-url -- rejected request targets use loopback only + for (const requestTarget of ['http://127.0.0.1:1/session', '//127.0.0.1:1/session']) { + const status = await new Promise((resolve, reject) => { + const request = http.request( + { hostname: '127.0.0.1', port: new URL(unconfirmedProxy.url).port, path: requestTarget }, + (response) => { + response.resume(); + resolve(response.statusCode); + }, + ); + request.once('error', reject); + request.end(); + }); + assert.equal(status, 400); + } + await run('lane', { input: 'accessible', report: unconfirmedReport, url: unconfirmedProxy.url }); + throw new Error('An accessible lane must not run against a driver that does not confirm its input policy.'); } catch (error) { if (!(error instanceof Error) || !(error instanceof WdioStoryTestRunError) || !('results' in error) || !Array.isArray(error.results)) throw error; - unmaskedLane = { results: laneResults(error.results), message: error.message }; + unconfirmedLane = { results: laneResults(error.results), message: error.message }; + } finally { + await unconfirmedProxy.close(); } - const unmaskedLaneResult = await unmaskedReport.finish(); + const unconfirmedLaneResult = await unconfirmedReport.finish(); + // Receipts hold every declared case to what it says, in compatibility mode as in a lane, and the report keeps them. + const receiptResults = async (input) => { + const report = new DesktopRunReporter({ + artifactsRoot: path.join(root, 'receipt-reports'), + ...(input ? { lane: { input, source: 'option', policyEnforced: true } } : {}), + summary: false, + }); + let results; + try { + await run('receipts', { report, ...(input ? { input } : {}) }); + throw new Error('Expected undeclared input to fail.'); + } catch (error) { + if ( + !(error instanceof Error) || + !(error instanceof WdioStoryTestRunError) || + !('results' in error) || + !Array.isArray(error.results) + ) + throw error; + results = error.results; + } + const reported = await report.finish(); + return results.map(({ testName, status, input: delivered, error }) => { + assert.deepEqual(reported.tests.find(({ title }) => title.endsWith(` / ${testName}`))?.attempts[0].input, delivered); + return { testName, status, input: delivered, ...(error ? { error } : {}) }; + }); + }; + const receipts = { compatibility: await receiptResults(), accessible: await receiptResults('accessible') }; const maskedHarness = await createDesktopDriverStoryHarness(manifest, { features: { keyboard: false, physicalClick: false, wheel: false }, }); @@ -394,14 +460,16 @@ async function main() { accessible: accessibleLane, physical: physicalLane, compatibility: compatibilityLane, - unmasked: { - ...unmaskedLane, - status: unmaskedLaneResult.status, - failed: unmaskedLaneResult.counts.failed, - excluded: unmaskedLaneResult.selection?.excluded, - reported: unmaskedLaneResult.failures.map(({ error }) => error.message), + unmasked: unmaskedLane, + unconfirmed: { + ...unconfirmedLane, + status: unconfirmedLaneResult.status, + failed: unconfirmedLaneResult.counts.failed, + excluded: unconfirmedLaneResult.selection?.excluded, + reported: unconfirmedLaneResult.failures.map(({ error }) => error.message), }, }, + receipts, platforms, namedFailureResults: namedFailureResults.map(({ testName, status }) => ({ testName, status })), namedTimeout, @@ -418,6 +486,61 @@ async function main() { } } +/** Forwards to a driver but drops the input policy from each new session, as a driver from before enforcement would. */ +async function createUnconfirmingProxy(url) { + const upstream = new URL(url); + if (upstream.protocol !== 'http:' || upstream.hostname !== '127.0.0.1') { + throw new Error('The test proxy requires a loopback HTTP driver.'); + } + const server = http.createServer(async (request, response) => { + if (!request.url?.startsWith('/') || request.url.startsWith('//')) { + request.resume(); + response.writeHead(400); + response.end(); + return; + } + const chunks = []; + for await (const chunk of request) chunks.push(chunk); + const body = Buffer.concat(chunks); + const reply = await new Promise((resolve, reject) => { + const forwarded = http.request( + { + hostname: '127.0.0.1', + port: upstream.port, + path: request.url, + method: request.method, + headers: { 'content-type': 'application/json' }, + }, + resolve, + ); + forwarded.once('error', reject); + forwarded.end(body); + }); + const replyChunks = []; + for await (const chunk of reply) replyChunks.push(chunk); + let text = Buffer.concat(replyChunks).toString('utf8'); + const status = reply.statusCode ?? 502; + if (request.method === 'POST' && request.url === '/session' && status >= 200 && status < 300) { + const created = JSON.parse(text); + delete created.value.capabilities['furn:inputMode']; + text = JSON.stringify(created); + } + response.writeHead(status, { 'content-type': 'application/json' }); + response.end(text); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve(undefined))); + const { port } = /** @type {import('node:net').AddressInfo} */ (server.address()); + return { + // eslint-disable-next-line @microsoft/sdl/no-insecure-url -- test proxy only binds to loopback + url: `http://127.0.0.1:${port}`, + close: () => + new Promise((resolve) => { + server.closeAllConnections(); + server.close(resolve); + }), + }; +} + /** @returns {Promise<{code: number | null, stdout: string, stderr: string}>} */ function runSmokeControl(packageRoot, cwd, manifestPath) { return new Promise((resolve, reject) => { diff --git a/packages/agentic/storybook-desktop/src/testing/wdioRunner.test.ts b/packages/agentic/storybook-desktop/src/testing/wdioRunner.test.ts index dc74dfdf44..0365095295 100644 --- a/packages/agentic/storybook-desktop/src/testing/wdioRunner.test.ts +++ b/packages/agentic/storybook-desktop/src/testing/wdioRunner.test.ts @@ -23,7 +23,7 @@ test('executes isolated inline code and releases sessions after assertion failur !diagnostics.includes('wdio "fixture-button--fail" during execution') || !diagnostics.includes('wdio "fixture-button--navigation-failure" during navigation') || !diagnostics.includes('Storybook context did not settle.') || - !diagnostics.includes('wdio "fixture-button--lane / accessible only" during lane verification') || + !diagnostics.includes('wdio "fixture-button--lane / accessible only" during input policy verification') || diagnostics.includes('Navigation refused before dispatch') || output.includes('[storybook] OK wdio "fixture-button--exit-zero"') ) { @@ -79,15 +79,54 @@ test('executes isolated inline code and releases sessions after assertion failur { testName: 'either lane', status: 'passed' }, { testName: 'physical only', status: 'unsupported' }, ], - // A driver that does not report physical input disabled stops the lane at its first case, reported once. - unmasked: { + // The driver holds the policy, so an accessible lane runs on a driver with physical input enabled. + unmasked: [ + { testName: 'accessible only', status: 'passed' }, + { testName: 'either lane', status: 'passed' }, + ], + // A driver that does not confirm the policy stops the lane at its first case, reported once. + unconfirmed: { results: [{ testName: 'accessible only', status: 'failed' }], status: 'failed', failed: 1, excluded: { platform: 0, scope: 0, input: 1 }, - reported: [expect.stringContaining('driver --input accessible')], + reported: [expect.stringContaining('did not confirm the accessible input policy (it reported none)')], }, }, + receipts: { + // Compatibility mode delivers what the host has, and still holds each declared case to its declaration. + compatibility: [ + { testName: 'declared click', status: 'passed', input: { receipts: { 'physical-click': 1 } } }, + { + testName: 'undeclared click', + status: 'failed', + input: { receipts: { 'physical-click': 1 }, undeclared: ['physical-click'] }, + error: 'The test delivered input it does not declare: physical-click (declare physical-click or click).', + }, + { testName: 'unused keyboard', status: 'passed', input: { receipts: {}, unused: ['keyboard'] } }, + { + testName: 'typing', + status: 'failed', + input: { receipts: { keyboard: 1 }, undeclared: ['keyboard'] }, + error: 'The test delivered input it does not declare: keyboard (declare keyboard).', + }, + ], + // The accessible lane clicks semantically and refuses typing before it is delivered, so nothing is attributed to it. + accessible: [ + { testName: 'declared click', status: 'passed', input: { receipts: { 'accessibility-click': 1 } } }, + { + testName: 'undeclared click', + status: 'failed', + input: { receipts: { 'accessibility-click': 1 }, undeclared: ['accessibility-click'] }, + }, + { + testName: 'typing', + status: 'failed', + input: { receipts: {} }, + error: expect.stringContaining('never delivers'), + }, + ], + }, platforms: Object.fromEntries( ['macos', 'windows', 'win32'].map((platform) => [ platform,