Desktop testing: enforce session input policy

.changeset/session-input-policy.md

@@ -0,0 +1,7 @@
+---
+'@fluentui-react-native/desktop-driver': patch
+'@fluentui-react-native/desktop-test': patch
+'@fluentui-react-native/storybook-desktop': patch
+---
+
+Enforce input policy per WebDriver session and report case-level input mechanism receipts.

packages/agentic/desktop-driver/native/PROTOCOL.md

@@ -283,6 +283,39 @@ one input turn and refuses a display name the element does not publish
 exactly once, so a key never reaches a different action than the one the
 agent named.
 
+### Clear mechanisms
+
+The mutating `clear` command takes `{ "elementId": "<id>" }`. Helpers that
+advertise `strictClear` in `hello.features` and in the `features` map of the
+`probe` result also accept an optional `mechanism` of `"semantic"` or
+`"keyboard"`, and answer every successful clear with
+`{ "mechanism": "semantic" | "keyboard" }`, naming the mechanism that cleared
+the element:
+
+- Without `mechanism`, the helper writes an empty value through the platform
+  accessibility value (macOS `AXValue`, the Windows UI Automation Value
+  pattern) and, when the element has no writable value or the write is
+  refused, falls back to focusing the element and pressing select-all and
+  Delete.
+- `semantic` uses only the value write and never falls back to the keyboard.
+  An element without a writable value fails `element-not-interactable` with
+  effect `none` before anything is dispatched, and a broken element is
+  `stale-element`. A write that was dispatched and failed settles as any other
+  mutation does: a refusal that proves the write never acted is `none`, any
+  other failure is `unknown`, and a write that never settles is
+  `mutation-indeterminate`.
+- `keyboard` skips the value write and clears only with the keyboard, behind
+  the same input authority, session, focus, and physical input lock checks as
+  the fallback. A check that refuses, such as `input-unavailable`, dispatches
+  nothing and is `none`.
+
+Any other `mechanism`, including one that is not a string, fails
+`invalid-params` with effect `none`. Node never sends `mechanism` unless the
+feature is negotiated, and treats a result without a recognized `mechanism`,
+such as the `null` an older helper returns, as not saying which mechanism
+cleared. The parameter, result, and feature are additive and do not change the
+wire version.
+
 ### Bounded observation
 
 Helpers that advertise `boundedObservation` in `hello.features` and in the

packages/agentic/desktop-driver/native/macos/README.md

@@ -289,6 +289,17 @@ responsible process and an unlocked interactive Aqua session. Missing authority
 returns `input-unavailable`; the provider never substitutes unrelated
 accessibility activation and reports physical success.
 
+The helper advertises `strictClear`. `clear` writes an empty `AXValue` when
+`AXUIElementIsAttributeSettable` reports the value settable, and otherwise
+activates the application, focuses the element, and presses Command-A and
+Delete under the physical input lock. A request may name `mechanism`:
+`semantic` only writes `AXValue` and fails `element-not-interactable` with
+effect `none` when the value is not settable, and `keyboard` skips the write
+and needs the same physical input authority as the fallback. Every successful
+clear returns `{ "mechanism": "semantic" | "keyboard" }`. The `probe` feature
+is `true` even without Accessibility trust, because each mechanism reports its
+own refusal.
+
 A locked screen or a login session that is not the active console prevents
 applications from publishing complete accessibility windows and from receiving
 physical input. Launch, attach, window discovery, and new pointer, keyboard,

packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Accessibility.swift

@@ -378,11 +378,27 @@ final class AccessibilityEngine {
     }
   }
 
-  func tryClearValue(_ elementID: String) throws -> Bool {
+  /// Writes an empty `AXValue` and reports whether that cleared the element. Without `strict`, an element whose value
+  /// cannot be written answers false so the caller can clear with the keyboard. With it, the method never answers
+  /// false: such an element fails `element-not-interactable` before anything is dispatched, and a failed check or
+  /// write fails with the effect the ledger recorded.
+  func tryClearValue(_ elementID: String, strict: Bool = false) throws -> Bool {
     let record = try requireRecord(elementID)
     try validateLive(record)
     var settable: DarwinBoolean = false
     let settableError = AXUIElementIsAttributeSettable(record.element, kAXValueAttribute as CFString, &settable)
+    if strict {
+      if settableError == .attributeUnsupported || settableError == .noValue
+        || (settableError == .success && !settable.boolValue)
+      {
+        try fail(ErrorCode.notInteractable, "The element does not expose a writable value to clear.")
+      }
+      try throwAX(
+        settableError,
+        operation: "Checking whether the element value can be set",
+        staleMessage: "Element \"\(elementID)\" is no longer available."
+      )
+    }
     if settableError == .attributeUnsupported || settableError == .noValue || !settable.boolValue {
       return false
     }
@@ -398,6 +414,9 @@ final class AccessibilityEngine {
     if error == .invalidUIElement {
       try fail(ErrorCode.staleElement, "Element \"\(elementID)\" is no longer available.")
     }
+    if strict {
+      try throwAX(error, operation: "Clearing the element value")
+    }
     return error == .success
   }
 

packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Driver.swift

@@ -198,8 +198,9 @@ final class Driver {
       )
       return CommandResult()
     case "clear":
-      try clear(elementID: try requireString(params, "elementId"), token: token)
-      return CommandResult()
+      let elementID = try requireString(params, "elementId")
+      let mechanism = try clearMechanism(params)
+      return CommandResult(result: ["mechanism": try clear(elementID: elementID, mechanism: mechanism, token: token)])
     case "sendKeys":
       guard let text = params["text"] as? String else {
         try fail(ErrorCode.invalidParams, "The sendKeys request requires a string \"text\".")
@@ -296,6 +297,7 @@ final class Driver {
         "physicalClick": trusted,
         "screenshot": screenshot,
         "setWindowRect": trusted,
+        "strictClear": true,
         "wheel": trusted,
         "windowEvents": trusted,
       ],
@@ -520,16 +522,34 @@ final class Driver {
     }
   }
 
-  private func clear(elementID: String, token: CancellationToken) throws {
+  /// The clear mechanism a request names, or nil when the helper may choose. Anything else is refused before dispatch.
+  private func clearMechanism(_ params: JSONObject) throws -> String? {
+    guard let value = params["mechanism"] else {
+      return nil
+    }
+    guard let mechanism = value as? String, mechanism == "semantic" || mechanism == "keyboard" else {
+      try fail(ErrorCode.invalidParams, "The clear mechanism must be \"semantic\" or \"keyboard\".")
+    }
+    return mechanism
+  }
+
+  /// Clears the element and returns the mechanism that did. Without a requested mechanism the value write comes first
+  /// and the keyboard is its fallback; a requested mechanism is the only one used.
+  private func clear(elementID: String, mechanism: String?, token: CancellationToken) throws -> String {
     let window = try windowForElement(elementID)
     _ = try accessibility.snapshot(elementID: elementID, window: window, token: token)
-    if try accessibility.tryClearValue(elementID) {
-      return
+    if mechanism != "keyboard", try accessibility.tryClearValue(elementID, strict: mechanism == "semantic") {
+      return "semantic"
+    }
+    guard mechanism != "semantic" else {
+      try fail(ErrorCode.internalError, "The semantic clear ended without clearing or failing.")
     }
     guard input.physicalInputAvailable() else {
       try fail(
         ErrorCode.inputUnavailable,
-        "The element does not expose a writable value and physical input requires Accessibility permission."
+        mechanism == "keyboard"
+          ? "Physical keyboard input requires Accessibility permission in System Settings > Privacy & Security > Accessibility."
+          : "The element does not expose a writable value and physical input requires Accessibility permission."
       )
     }
     try withPhysicalInput(token: token) {
@@ -538,6 +558,7 @@ final class Driver {
       try token.wait(milliseconds: 16)
       try input.pressCommandADelete(token: token)
     }
+    return "keyboard"
   }
 
   private func sendKeys(elementID: String, text: String, token: CancellationToken) throws {

packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/Host.swift

@@ -31,6 +31,7 @@ func createHello() -> JSONObject {
       "releaseActions",
       "setWindowRect",
       "source",
+      "strictClear",
       "tree",
       "wheel",
       "windowEvents",

packages/agentic/desktop-driver/native/macos/Sources/DesktopDriverHost/SelfTest.swift

@@ -772,6 +772,26 @@ final class SelfTestRunner {
       actions?.code == ErrorCode.invalidParams && actions?.effect == .noEffect,
       "a malformed input sequence reports no effect"
     )
+    for mechanism: Any in ["pointer", "", 1, NSNull()] {
+      residue.dispatching()
+      let clear = failure("clear", ["elementId": "element-missing", "mechanism": mechanism]) as? HelperError
+      check(
+        clear?.code == ErrorCode.invalidParams && clear?.effect == .noEffect,
+        "a clear naming an unknown mechanism (\(mechanism)) is refused before dispatch"
+      )
+    }
+    for mechanism in ["semantic", "keyboard"] {
+      let clear = failure("clear", ["elementId": "element-missing", "mechanism": mechanism]) as? HelperError
+      check(
+        clear?.code == ErrorCode.staleElement && clear?.effect == .noEffect,
+        "a \(mechanism) clear of a missing element reports no effect"
+      )
+    }
+    let probe = try? driver.execute(command: "probe", params: [:], token: token).result as? JSONObject
+    check(
+      (probe?["features"] as? JSONObject)?["strictClear"] as? Bool == true,
+      "probe reports that clear honors a requested mechanism"
+    )
     let read = failure("windows") as? HelperError
     check(read != nil && read?.effect == nil, "a read-only command does not claim an effect")
     let unsupported = failure("not-a-command") as? HelperError
@@ -931,6 +951,7 @@ final class SelfTestRunner {
       (hello["features"] as? [String])?.contains("performAccessibilityAction") == true,
       "hello advertises performing accessibility actions"
     )
+    check((hello["features"] as? [String])?.contains("strictClear") == true, "hello advertises strict clear")
   }
 
   private func testAccessibilityActionNames() {

packages/agentic/desktop-driver/native/windows/README.md

@@ -172,6 +172,19 @@ accessibility clicks. Neither names nor keys enter the native command journal.
 The `observe` `actions` fact uses cached pattern-availability properties, so
 listing actions adds no per-node provider calls to the bounded walk.
 
+### Clear mechanisms
+
+The helper advertises `strictClear`. `clear` writes an empty value through the
+UIA Value pattern when the element exposes one that is not read-only, and
+otherwise presses Ctrl+A and Delete after focusing the element. A request may
+name `mechanism`: `semantic` only writes the value and fails
+`element-not-interactable` with effect `none` when the pattern is missing or
+read-only, and `keyboard` skips the write and takes the same input authority,
+focus check, and physical input lock as the fallback. The value write uses the
+bounded mutation worker, effect ledger, and lease-scoped quarantine, so a
+`semantic` write that never settles is `mutation-indeterminate`. Every
+successful clear returns `{ "mechanism": "semantic" | "keyboard" }`.
+
 ### Window events and identity
 
 Window lifecycle comes from out-of-context `SetWinEventHook` notifications
@@ -213,7 +226,8 @@ point. A `SendInput` call that injects fewer events than requested fails as
 
 `click` mode `auto` falls back to accessibility activation when physical input
 is unavailable, and an explicit `physical` request reports `input-unavailable`.
-Element `sendKeys` and the physical `clear` fallback activate the target window
+Element `sendKeys` and the keyboard `clear` (the fallback, or a `keyboard`
+mechanism) activate the target window
 and verify keyboard focus before sending, which is stricter than the macOS
 `clear`; `performActions` activates only when the foreground belongs to another
 application. The helper has no `uiAccess` manifest, and the occlusion check

packages/agentic/desktop-driver/native/windows/src/automation.cpp

@@ -1422,17 +1422,35 @@ void Automation::PerformAccessibilityAction(const std::string& elementId, const
   }
 }
 
-bool Automation::TryClearValue(const std::string& elementId) {
+bool Automation::TryClearValue(const std::string& elementId, bool strict) {
   const ElementRecord& record = RequireRecord(elementId);
   winrt::com_ptr<IUIAutomationValuePattern> value;
-  if (FAILED(record.element->GetCurrentPatternAs(UIA_ValuePatternId, IID_PPV_ARGS(value.put()))) || !value) {
-    return false;
+  const HRESULT query = record.element->GetCurrentPatternAs(UIA_ValuePatternId, IID_PPV_ARGS(value.put()));
+  if (FAILED(query) || !value) {
+    if (!strict) {
+      return false;
+    }
+    if (query == UIA_E_ELEMENTNOTAVAILABLE) {
+      Fail(kErrorStaleElement, "The element is no longer available.");
+    }
+    if (SUCCEEDED(query) || query == UIA_E_NOTSUPPORTED || query == E_NOINTERFACE || query == E_NOTIMPL ||
+        query == E_FAIL) {
+      Fail(kErrorNotInteractable, "The element does not expose a writable value to clear.");
+    }
+    FailHresult(kErrorAutomationFailed, "Reading the element value pattern", query);
   }
   const auto readOnly = std::make_shared<BOOL>(TRUE);
   const HRESULT state = BoundedRead([value, readOnly]() { return value->get_CurrentIsReadOnly(readOnly.get()); },
                                     "Reading the element value state");
   if (FAILED(state) || *readOnly != FALSE) {
-    return false;
+    if (!strict) {
+      return false;
+    }
+    if (state == UIA_E_ELEMENTNOTAVAILABLE) {
+      Fail(kErrorStaleElement, "The element is no longer available.");
+    }
+    CheckHresult(kErrorAutomationFailed, "Reading the element value state", state);
+    Fail(kErrorNotInteractable, "The element's value is read-only, so it cannot be cleared.");
   }
   const HRESULT result = CountedMutation(
       [value]() {
@@ -1442,6 +1460,13 @@ bool Automation::TryClearValue(const std::string& elementId) {
         return assigned;
       },
       "Clearing the element value");
+  if (strict) {
+    if (result == UIA_E_ELEMENTNOTAVAILABLE) {
+      Fail(kErrorStaleElement, "The element is no longer available.");
+    }
+    CheckHresult(result == UIA_E_ELEMENTNOTENABLED ? kErrorNotInteractable : kErrorAutomationFailed,
+                 "Clearing the element value", result);
+  }
   return SUCCEEDED(result);
 }
 

packages/agentic/desktop-driver/native/windows/src/automation.h

@@ -103,7 +103,13 @@ class Automation {
   void AccessibilityClick(const std::string& elementId, const CancellationToken& token);
   json::Value AccessibilityActions(const std::string& elementId, const CancellationToken& token);
   void PerformAccessibilityAction(const std::string& elementId, const std::string& key, const CancellationToken& token);
-  bool TryClearValue(const std::string& elementId);
+  // Writes an empty value through the Value pattern and reports whether that
+  // cleared the element. Without `strict`, an element without a writable value
+  // or a refused write returns false so the caller can clear with the
+  // keyboard. With it, the method never returns false: such an element fails
+  // as not interactable before anything is dispatched, and a failed write
+  // fails with the effect the ledger recorded.
+  bool TryClearValue(const std::string& elementId, bool strict = false);
   bool HasKeyboardFocus(const std::string& elementId);
 
   void ForgetWindow(const std::string& windowId);

packages/agentic/desktop-driver/native/windows/src/driver.cpp

@@ -41,6 +41,19 @@ std::string RequireString(const json::Value& params, std::string_view key) {
   return value->AsString();
 }
 
+// The clear mechanism a request names, or empty when the helper may choose.
+// Anything else is refused before dispatch.
+std::string RequireClearMechanism(const json::Value& params) {
+  const json::Value* mechanism = params.Find("mechanism");
+  if (mechanism == nullptr) {
+    return {};
+  }
+  if (!mechanism->IsString() || (mechanism->AsString() != "semantic" && mechanism->AsString() != "keyboard")) {
+    Fail(kErrorInvalidParams, "The clear mechanism must be \"semantic\" or \"keyboard\".");
+  }
+  return mechanism->AsString();
+}
+
 int ActionButton(const json::Value& action) {
   const json::Value* button = action.Find("button");
   if (button == nullptr || !button->IsNumber()) {
@@ -274,6 +287,7 @@ json::Value Driver::ProbeResult(const json::Value& params) {
   features.Set("physicalClick", json::Value::Bool(physical));
   features.Set("screenshot", json::Value::Bool(screenshot));
   features.Set("setWindowRect", json::Value::Bool(true));
+  features.Set("strictClear", json::Value::Bool(true));
   features.Set("wheel", json::Value::Bool(physical));
   features.Set("windowEvents", json::Value::Bool(true));
 
@@ -769,17 +783,23 @@ CommandResult Driver::ExecuteCommand(const std::string& command, const json::Val
   }
   if (command == "clear") {
     const std::string elementId = RequireString(params, "elementId");
+    const std::string mechanism = RequireClearMechanism(params);
     RequireLiveElement(elementId, token);
     const MutationScope scope(LeaseIdForElement(elementId));
-    if (!automation_.TryClearValue(elementId)) {
+    const bool semantic = mechanism != "keyboard" && automation_.TryClearValue(elementId, mechanism == "semantic");
+    if (!semantic && mechanism == "semantic") {
+      Fail(kErrorInternal, "The semantic clear ended without clearing or failing.");
+    }
+    if (!semantic) {
       RunPhysical(token, [&]() {
         const HWND window = FocusForKeyboard(elementId, token);
         const KeyboardGuardScope guard(input_, KeyboardGuardFor(window));
         input_.PressChord({VK_CONTROL, 'A'}, token);
         input_.PressChord({VK_DELETE}, token);
       });
     }
-    result.result = json::Value::Null();
+    result.result = json::Value::Object();
+    result.result.Set("mechanism", json::Value::String(std::string(semantic ? "semantic" : "keyboard")));
     return result;
   }
   if (command == "sendKeys") {

packages/agentic/desktop-driver/native/windows/src/host.cpp

@@ -51,7 +51,7 @@ json::Value CreateHello() {
   for (const char* feature : {"accessibilityActions", "accessibilityClick", "performAccessibilityAction",
                               "activeElement", "boundedObservation", "capture", "elementCapture",
                               "find", "hitTest", "keyboard", "launch", "attach", "pointer", "probe", "releaseActions",
-                              "setWindowRect", "source", "tree", "wheel", "windowEvents"}) {
+                              "setWindowRect", "source", "strictClear", "tree", "wheel", "windowEvents"}) {
     features.Append(json::Value::String(std::string(feature)));
   }
   hello.Set("features", features);