From f5a90cdda9b429b72d9ef9b1429211a114ef71ab Mon Sep 17 00:00:00 2001 From: Jason Morse Date: Fri, 6 Nov 2026 14:47:43 -0800 Subject: [PATCH 1/6] WDIO descriptor and desktopIt helper --- .changeset/input-lanes.md | 6 + .../desktop-driver/src/authoring/index.ts | 9 + .../src/authoring/input.test.ts | 32 +++ .../desktop-driver/src/authoring/input.ts | 167 +++++++++++++++ .../hosts/native/NativeDesktopHost.test.ts | 14 ++ .../src/hosts/native/NativeDesktopHost.ts | 12 +- packages/agentic/desktop-driver/src/index.ts | 9 + .../agentic/desktop-driver/src/storybook.ts | 8 +- .../config/transform-story-tests.cjs | 83 ++++++++ .../config/transform-story-tests.d.cts | 3 +- .../storybook-desktop/references/commands.md | 8 + .../references/story-tests.md | 45 +++- .../src/cli/DesktopStorybookCli.ts | 96 +++++++-- .../src/cli/createDesktopStorybookCommand.ts | 17 +- .../storybook-desktop/src/cli/smokeTests.ts | 4 +- .../storybook-desktop/src/config/commands.ts | 1 + .../src/config/transformStoryTests.test.ts | 10 + .../storybook-desktop/src/config/wdio.test.ts | 16 ++ .../storybook-desktop/src/config/wdio.ts | 23 +- .../src/driver/driverManifest.ts | 4 +- .../src/driver/storyManifest.ts | 12 +- .../src/testing/desktopIt.ts | 11 + .../src/testing/extractWdioTests.test.ts | 31 +++ .../src/testing/extractWdioTests.ts | 7 +- .../storybook-desktop/src/testing/index.ts | 3 +- .../src/testing/runWdioTests.test.ts | 68 ++++++ .../src/testing/runWdioTests.ts | 197 +++++++++++++++--- .../storybook-desktop/src/testing/types.ts | 5 +- .../src/testing/wdio.types.test.ts | 14 +- 29 files changed, 844 insertions(+), 71 deletions(-) create mode 100644 .changeset/input-lanes.md create mode 100644 packages/agentic/desktop-driver/src/authoring/input.test.ts create mode 100644 packages/agentic/desktop-driver/src/authoring/input.ts create mode 100644 packages/agentic/storybook-desktop/src/testing/desktopIt.ts create mode 100644 packages/agentic/storybook-desktop/src/testing/runWdioTests.test.ts diff --git a/.changeset/input-lanes.md b/.changeset/input-lanes.md new file mode 100644 index 0000000000..4c06b6b268 --- /dev/null +++ b/.changeset/input-lanes.md @@ -0,0 +1,6 @@ +--- +'@fluentui-react-native/desktop-driver': patch +'@fluentui-react-native/storybook-desktop': patch +--- + +Add modality-aware desktop test descriptors and Storybook input-lane selection. diff --git a/packages/agentic/desktop-driver/src/authoring/index.ts b/packages/agentic/desktop-driver/src/authoring/index.ts index abc515c0d9..0fd5cd6695 100644 --- a/packages/agentic/desktop-driver/src/authoring/index.ts +++ b/packages/agentic/desktop-driver/src/authoring/index.ts @@ -14,6 +14,15 @@ export { desktopStoryPlatforms, validateDesktopStoryTests, } from './storyTests.js'; +export { desktopRequirements, matchesDesktopPlatforms, resolveDesktopInput, validateDesktopCaseOptions } from './input.js'; +export type { + DesktopCaseOptions, + DesktopInput, + DesktopInputLane, + DesktopPlatforms, + DesktopRequirement, + ResolvedDesktopCaseOptions, +} from './input.js'; export type { DesktopStoryCapability, DesktopStoryExpectation, diff --git a/packages/agentic/desktop-driver/src/authoring/input.test.ts b/packages/agentic/desktop-driver/src/authoring/input.test.ts new file mode 100644 index 0000000000..1c02abba19 --- /dev/null +++ b/packages/agentic/desktop-driver/src/authoring/input.test.ts @@ -0,0 +1,32 @@ +import { matchesDesktopPlatforms, resolveDesktopInput, validateDesktopCaseOptions } from './input.js'; + +describe('desktop input descriptors', () => { + test.each([ + [undefined, [], 'both'], + [undefined, ['click'], 'both'], + [undefined, ['physical-click'], 'physical'], + [undefined, ['accessibility-click'], 'accessible'], + ['accessible', [], 'accessible'], + ['physical', ['click'], 'physical'], + ] as const)('derives input from %s and %s', (input, requires, expected) => { + expect(resolveDesktopInput(input, requires)).toBe(expected); + }); + + test('rejects mixed transports and explicit widening', () => { + expect(() => resolveDesktopInput(undefined, ['keyboard', 'accessibility-click'])).toThrow('split the case'); + expect(() => resolveDesktopInput('both', ['physical-click'])).toThrow('cannot widen'); + expect(() => resolveDesktopInput('physical', ['accessibility-click'])).toThrow('cannot widen'); + }); + + test('validates platform filters and negation', () => { + expect(matchesDesktopPlatforms(['!win32'], 'windows')).toBe(true); + expect(matchesDesktopPlatforms(['!win32'], 'win32')).toBe(false); + expect(() => validateDesktopCaseOptions({ platforms: ['macos', '!win32'] })).toThrow('known'); + expect(() => validateDesktopCaseOptions({ platforms: ['!macos', '!windows', '!win32'] })).toThrow('every platform'); + }); + + test('rejects duplicate and unknown requirements', () => { + expect(() => validateDesktopCaseOptions({ requires: ['click', 'click'] })).toThrow('duplicate'); + expect(() => validateDesktopCaseOptions({ requires: ['input'] })).toThrow('unsupported requirement'); + }); +}); diff --git a/packages/agentic/desktop-driver/src/authoring/input.ts b/packages/agentic/desktop-driver/src/authoring/input.ts new file mode 100644 index 0000000000..ee0e353c51 --- /dev/null +++ b/packages/agentic/desktop-driver/src/authoring/input.ts @@ -0,0 +1,167 @@ +import type { DesktopEndpoint } from '../protocol/types.js'; + +export const desktopRequirements = [ + 'accessibility-actions', + 'accessibility-click', + 'click', + 'element-screenshot', + 'focus', + 'keyboard', + 'physical-click', + 'screenshot', + 'set-window-rect', + 'wheel', +] as const; + +export type DesktopRequirement = (typeof desktopRequirements)[number]; +export type DesktopInput = 'accessible' | 'physical' | 'both'; +export type DesktopInputLane = Exclude; +export type DesktopPlatforms = + | readonly [DesktopEndpoint, ...DesktopEndpoint[]] + | readonly [`!${DesktopEndpoint}`, ...`!${DesktopEndpoint}`[]]; + +export type DesktopCaseOptions = { + input?: DesktopInput; + requires?: readonly DesktopRequirement[]; + platforms?: DesktopPlatforms; + scope?: Readonly>; + unsupported?: { platforms: DesktopPlatforms; reason: string; issue?: string }; + quarantine?: { + issue: string; + reason: string; + expires: string; + platforms?: DesktopPlatforms; + input?: DesktopInputLane; + }; +}; + +export type ResolvedDesktopCaseOptions = DesktopCaseOptions & { + input: DesktopInput; + inputSource: 'declared' | 'derived'; +}; + +const physicalRequirements = new Set(['physical-click', 'keyboard', 'wheel']); +const accessibleRequirements = new Set(['accessibility-click']); +const endpoints = ['macos', 'windows', 'win32'] as const; +const requirementSet = new Set(desktopRequirements); + +export function resolveDesktopInput(input?: DesktopInput, requires: readonly DesktopRequirement[] = []): DesktopInput { + const physical = requires.some((requirement) => physicalRequirements.has(requirement)); + const accessible = requires.some((requirement) => accessibleRequirements.has(requirement)); + if (physical && accessible) { + throw new TypeError('A desktop test cannot require both physical and accessible input; split the case.'); + } + const derived = physical ? 'physical' : accessible ? 'accessible' : 'both'; + if (input !== undefined && derived !== 'both' && input !== derived) { + throw new TypeError(`input "${input}" cannot widen the modality derived from requires ("${derived}").`); + } + return input ?? derived; +} + +export function validateDesktopCaseOptions(value: unknown, source = 'desktop test options'): ResolvedDesktopCaseOptions { + const options = object(value, source); + exactKeys(options, ['input', 'requires', 'platforms', 'scope', 'unsupported', 'quarantine'], source); + if (options.input !== undefined && !['accessible', 'physical', 'both'].includes(options.input as string)) { + throw new TypeError(`${source}.input must be accessible, physical, or both.`); + } + const requires: DesktopRequirement[] = []; + if (options.requires !== undefined) { + if (!Array.isArray(options.requires)) throw new TypeError(`${source}.requires must be an array.`); + const seen = new Set(); + for (const requirement of options.requires) { + if (typeof requirement !== 'string' || !requirementSet.has(requirement)) { + throw new TypeError(`${source}.requires contains an unsupported requirement.`); + } + if (seen.has(requirement)) throw new TypeError(`${source}.requires contains duplicate "${requirement}".`); + seen.add(requirement); + requires.push(requirement as DesktopRequirement); + } + } + validatePlatforms(options.platforms, `${source}.platforms`); + validateScope(options.scope, `${source}.scope`); + if (options.unsupported !== undefined) { + const unsupported = object(options.unsupported, `${source}.unsupported`); + exactKeys(unsupported, ['platforms', 'reason', 'issue'], `${source}.unsupported`); + validatePlatforms(unsupported.platforms, `${source}.unsupported.platforms`, true); + nonEmptyString(unsupported.reason, `${source}.unsupported.reason`); + if (unsupported.issue !== undefined) nonEmptyString(unsupported.issue, `${source}.unsupported.issue`); + } + if (options.quarantine !== undefined) { + const quarantine = object(options.quarantine, `${source}.quarantine`); + exactKeys(quarantine, ['issue', 'reason', 'expires', 'platforms', 'input'], `${source}.quarantine`); + nonEmptyString(quarantine.issue, `${source}.quarantine.issue`); + nonEmptyString(quarantine.reason, `${source}.quarantine.reason`); + if (typeof quarantine.expires !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(quarantine.expires)) { + throw new TypeError(`${source}.quarantine.expires must be an ISO date.`); + } + if (quarantine.input !== undefined && !['accessible', 'physical'].includes(quarantine.input as string)) { + throw new TypeError(`${source}.quarantine.input must be accessible or physical.`); + } + validatePlatforms(quarantine.platforms, `${source}.quarantine.platforms`); + } + const resolvedInput = resolveDesktopInput(options.input as DesktopInput | undefined, requires); + return { + ...(options.input === undefined ? {} : { input: options.input as DesktopInput }), + ...(options.requires === undefined ? {} : { requires }), + ...(options.platforms === undefined ? {} : { platforms: options.platforms as DesktopPlatforms }), + ...(options.scope === undefined ? {} : { scope: options.scope as DesktopCaseOptions['scope'] }), + ...(options.unsupported === undefined ? {} : { unsupported: options.unsupported as DesktopCaseOptions['unsupported'] }), + ...(options.quarantine === undefined ? {} : { quarantine: options.quarantine as DesktopCaseOptions['quarantine'] }), + input: resolvedInput, + inputSource: options.input === undefined ? 'derived' : 'declared', + }; +} + +export function matchesDesktopPlatforms(platforms: DesktopPlatforms | undefined, endpoint: DesktopEndpoint): boolean { + if (platforms === undefined) return true; + const negative = platforms[0].startsWith('!'); + const values = platforms.map((platform) => (negative ? platform.slice(1) : platform)); + return negative ? !values.includes(endpoint) : values.includes(endpoint); +} + +function validatePlatforms(value: unknown, source: string, required = false): void { + if (value === undefined && !required) return; + if (!Array.isArray(value) || value.length === 0) throw new TypeError(`${source} must be a non-empty platform list.`); + const negative = typeof value[0] === 'string' && value[0].startsWith('!'); + const seen = new Set(); + for (const entry of value) { + if (typeof entry !== 'string') throw new TypeError(`${source} contains an unsupported platform.`); + const platform = negative ? entry.slice(1) : entry; + if (entry.startsWith('!') !== negative || !endpoints.includes(platform as (typeof endpoints)[number])) { + throw new TypeError(`${source} must contain only known ${negative ? 'negated ' : ''}platforms.`); + } + if (seen.has(platform)) throw new TypeError(`${source} contains duplicate "${entry}".`); + seen.add(platform); + } + if (negative && seen.size === endpoints.length) throw new TypeError(`${source} cannot exclude every platform.`); +} + +function validateScope(value: unknown, source: string): void { + if (value === undefined) return; + const scope = object(value, source); + for (const [key, values] of Object.entries(scope)) { + if (!/^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)*$/.test(key)) throw new TypeError(`${source} contains invalid key "${key}".`); + if (!Array.isArray(values) || values.length === 0) throw new TypeError(`${source}.${key} must be a non-empty value list.`); + const seen = new Set(); + for (const item of values) { + if (typeof item !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(item)) { + throw new TypeError(`${source}.${key} contains an invalid value.`); + } + if (seen.has(item)) throw new TypeError(`${source}.${key} contains duplicate "${item}".`); + seen.add(item); + } + } +} + +function object(value: unknown, source: string): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) throw new TypeError(`${source} must be an object.`); + return value as Record; +} + +function exactKeys(value: Record, allowed: readonly string[], source: string): void { + for (const key of Object.keys(value)) if (!allowed.includes(key)) throw new TypeError(`${source} contains unknown field "${key}".`); +} + +function nonEmptyString(value: unknown, source: string): void { + if (typeof value !== 'string' || !value.trim()) throw new TypeError(`${source} must be a non-empty string.`); +} diff --git a/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.test.ts b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.test.ts index ddcf1c77c9..710571be26 100644 --- a/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.test.ts +++ b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.test.ts @@ -8,6 +8,7 @@ import { FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES, NativeDesktopHost, parseKeyedActions, + resolveDisabledInputFeatures, translateNativeError, } from './NativeDesktopHost'; @@ -20,6 +21,19 @@ describe('NativeDesktopHost accessibility action keys', () => { expect(parseKeyedActions({ actions: [], keys: [] })).toEqual([]); }); + describe('NativeDesktopHost input feature policy', () => { + test('unions explicit masks with the operator environment mask', () => { + const previous = process.env[FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES]; + process.env[FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES] = 'physicalClick,wheel'; + try { + expect([...resolveDisabledInputFeatures(['keyboard'])].sort()).toEqual(['keyboard', 'physicalClick', 'wheel']); + } finally { + if (previous === undefined) delete process.env[FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES]; + else process.env[FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES] = previous; + } + }); + }); + test.each([ ['no keys', { actions: ['AXPress'] }], ['lists of different lengths', { actions: ['AXPress', 'Toggle'], keys: ['k1'] }], diff --git a/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.ts b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.ts index 8bd1da92d2..4c33f0aeb4 100644 --- a/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.ts +++ b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.ts @@ -497,14 +497,14 @@ export function applyDisabledInputFeaturePolicy( return { ...hostInfo, features }; } -function resolveDisabledInputFeatures(configured?: readonly NativeDisabledInputFeature[]): ReadonlySet { - const values = - configured ?? - process.env[FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES] +export function resolveDisabledInputFeatures(configured?: readonly NativeDisabledInputFeature[]): ReadonlySet { + const values = [ + ...(configured ?? []), + ...(process.env[FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES] ?.split(',') .map((value) => value.trim()) - .filter(Boolean) ?? - []; + .filter(Boolean) ?? []), + ]; const allowed = new Set(['keyboard', 'physicalClick', 'wheel']); const resolved = new Set(); for (const value of values) { diff --git a/packages/agentic/desktop-driver/src/index.ts b/packages/agentic/desktop-driver/src/index.ts index 5c66f111c9..3e556374bc 100644 --- a/packages/agentic/desktop-driver/src/index.ts +++ b/packages/agentic/desktop-driver/src/index.ts @@ -32,6 +32,15 @@ export { desktopStoryPlatforms, validateDesktopStoryTests, } from './authoring/storyTests.js'; +export { desktopRequirements, matchesDesktopPlatforms, resolveDesktopInput, validateDesktopCaseOptions } from './authoring/input.js'; +export type { + DesktopCaseOptions, + DesktopInput, + DesktopInputLane, + DesktopPlatforms, + DesktopRequirement, + ResolvedDesktopCaseOptions, +} from './authoring/input.js'; export type { DesktopStoryCapability, DesktopStoryExpectation, diff --git a/packages/agentic/desktop-driver/src/storybook.ts b/packages/agentic/desktop-driver/src/storybook.ts index 4946ad486e..9aa50d9d49 100644 --- a/packages/agentic/desktop-driver/src/storybook.ts +++ b/packages/agentic/desktop-driver/src/storybook.ts @@ -1,4 +1,5 @@ import type { DesktopStoryTests } from './authoring/storyTests.js'; +import type { ResolvedDesktopCaseOptions } from './authoring/input.js'; import type { DesktopEndpoint } from './protocol/types.js'; export type DesktopStoryManifestEntry = { @@ -9,7 +10,12 @@ export type DesktopStoryManifestEntry = { tags: readonly string[]; title: string; tests?: DesktopStoryTests; - wdio?: { digest: string; exportName: string; testNames?: readonly string[] }; + wdio?: { + digest: string; + exportName: string; + testNames?: readonly string[]; + cases?: readonly { name: string; options: ResolvedDesktopCaseOptions }[]; + }; }; export type DesktopStoryManifest = { diff --git a/packages/agentic/storybook-desktop/config/transform-story-tests.cjs b/packages/agentic/storybook-desktop/config/transform-story-tests.cjs index aac57cf91a..bc47137db0 100644 --- a/packages/agentic/storybook-desktop/config/transform-story-tests.cjs +++ b/packages/agentic/storybook-desktop/config/transform-story-tests.cjs @@ -57,6 +57,55 @@ function resolveInitializer(value, seen = new Set(), bindings = new Set()) { return value; } +function importedDesktopIt(call) { + const callee = call.get('callee'); + if (!callee.isIdentifier()) fail(call, 'array tests must be direct desktopIt imports.'); + const binding = callee.scope.getBinding(callee.node.name); + const specifier = binding?.path; + const declaration = specifier?.parentPath; + if ( + !specifier?.isImportSpecifier() || + (specifier.node.imported.name ?? specifier.node.imported.value) !== 'desktopIt' || + !declaration?.isImportDeclaration() || + declaration.node.source.value !== '@fluentui-react-native/storybook-desktop/testing' || + declaration.node.importKind === 'type' || + specifier.node.importKind === 'type' + ) { + fail(call, 'array tests must use the desktopIt value imported from @fluentui-react-native/storybook-desktop/testing.'); + } +} + +function staticValue(value) { + value = unwrap(value); + if (value.isStringLiteral() || value.isBooleanLiteral() || value.isNullLiteral()) return value.node.value; + if (value.isNumericLiteral() && Number.isFinite(value.node.value)) return value.node.value; + if (value.isTemplateLiteral() && value.node.expressions.length === 0 && value.node.quasis.length === 1) { + return value.node.quasis[0].value.cooked; + } + if (value.isUnaryExpression({ operator: '-' }) && value.get('argument').isNumericLiteral()) { + return -value.node.argument.value; + } + if (value.isArrayExpression()) { + return value.get('elements').map((element) => { + if (!element?.node || element.isSpreadElement()) fail(element ?? value, 'descriptor arrays must contain static values.'); + return staticValue(element); + }); + } + if (value.isObjectExpression()) { + const result = {}; + for (const property of value.get('properties')) { + if (!property.isObjectProperty() || property.node.computed || property.node.shorthand) { + fail(property, 'descriptor objects require static key/value properties.'); + } + const key = keyName(property); + if (!key || Object.hasOwn(result, key)) fail(property, 'descriptor objects require unique static property names.'); + result[key] = staticValue(property.get('value')); + } + return result; + } + fail(value, 'descriptor options must be a strict JSON literal.'); +} + function collectCallbacks(value) { const requireCallback = (callback) => { if ((!callback.isArrowFunctionExpression() && !callback.isFunctionExpression()) || callback.node.generator) { @@ -64,6 +113,29 @@ function collectCallbacks(value) { } return callback; }; + if (value.isArrayExpression()) { + const callbacks = []; + const names = new Set(); + for (const item of value.get('elements')) { + if (!item?.node || item.isSpreadElement() || !item.isCallExpression()) { + fail(item ?? value, 'desktopIt arrays require direct calls and cannot contain holes or spreads.'); + } + importedDesktopIt(item); + const args = item.get('arguments'); + if (args.length < 2 || args.length > 3 || args.some((argument) => argument.isSpreadElement())) { + fail(item, 'desktopIt requires a name, optional static options, and an inline callback.'); + } + const name = args[0].isStringLiteral() ? args[0].node.value : undefined; + if (!name?.trim()) fail(args[0], 'desktopIt requires a non-empty literal test name.'); + if (names.has(name)) fail(args[0], `duplicate test name "${name}".`); + names.add(name); + const options = args.length === 2 ? {} : staticValue(args[1]); + if (!options || typeof options !== 'object' || Array.isArray(options)) fail(args[1], 'desktopIt options must be an object.'); + callbacks.push({ name, options, callback: requireCallback(unwrap(args[args.length - 1])) }); + } + if (!callbacks.length) fail(value, 'a desktopIt array must contain at least one test.'); + return callbacks; + } if (!value.isObjectExpression()) return [{ callback: requireCallback(value) }]; const callbacks = []; const names = new Set(); @@ -276,6 +348,17 @@ module.exports = () => ({ for (const { callback } of callbacks) compileWdioTest(callback, filename, state.file.code); } for (const { property } of tests.values()) if (!property.removed) property.remove(); + program.scope.crawl(); + for (const statement of program.get('body')) { + if (!statement.isImportDeclaration() || statement.node.source.value !== '@fluentui-react-native/storybook-desktop/testing') + continue; + for (const specifier of statement.get('specifiers')) { + if (!specifier.isImportSpecifier() || (specifier.node.imported.name ?? specifier.node.imported.value) !== 'desktopIt') continue; + const binding = statement.scope.getBinding(specifier.node.local.name); + if (!binding?.referencePaths.length) specifier.remove(); + } + if (!statement.node.specifiers.length) statement.remove(); + } }, }, }); diff --git a/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts b/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts index 8444ed0b64..67e6ed935c 100644 --- a/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts +++ b/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts @@ -5,12 +5,13 @@ declare function storyTests(): PluginObj; declare namespace storyTests { type Callback = { name?: string; + options?: Record; callback: NodePath; }; type StoryTest = { callbacks: readonly Callback[]; property: NodePath; - value: NodePath; + value: NodePath; }; function collectWdioTests(program: NodePath): ReadonlyMap; diff --git a/packages/agentic/storybook-desktop/references/commands.md b/packages/agentic/storybook-desktop/references/commands.md index 27e2975392..f60208c21c 100644 --- a/packages/agentic/storybook-desktop/references/commands.md +++ b/packages/agentic/storybook-desktop/references/commands.md @@ -28,6 +28,7 @@ only). The mode precedence is CLI flag, `DESKTOP_REPORTER`, config, then | `instance --` | Inspect the active instance's ports and target | | `test -- --list` | Discover authored executable story-test IDs | | `test -- --test ` | Run matching isolated story callbacks | +| `test -- --input accessible` | Select accessible and `both` descriptor cases | | `smoke -- --mode stories` | Build/bundle/launch, traverse stories, clean up | | `smoke -- --mode stories-and-tests` | Traverse stories and execute callbacks | @@ -60,6 +61,13 @@ Metro, generated runtime identity, and exact process cleanup. Windows RN Test App retains its required Metro port 8081; do not run concurrent Metro-backed Windows smokes. +Storybook test and stories-and-tests smoke commands accept +`--input accessible|physical`. The lane filters `desktopIt` cases by their +resolved input and target platform. `driver --input` records the default lane +for subsequent tests against that instance. Legacy callback collections are +not eligible for input-lane runs. Input lanes cannot be combined with an +explicit non-`auto` click mode. + The TypeScript API exposes the same operations through `DesktopStorybookCli`. Higher-level supervisors should inject their shared reporter, runner, and services so runs join one owned lifecycle. diff --git a/packages/agentic/storybook-desktop/references/story-tests.md b/packages/agentic/storybook-desktop/references/story-tests.md index 1fd21d7e2e..62662edbbd 100644 --- a/packages/agentic/storybook-desktop/references/story-tests.md +++ b/packages/agentic/storybook-desktop/references/story-tests.md @@ -7,11 +7,12 @@ owner_paths: # Author story tests -Story tests are statically extracted CSF `wdio` callbacks. Use top-level -functions or named callback collections with literal names; callbacks must not -close over module-local values. Import Node-only helpers through literal -dynamic `import()` inside the callback. Keep test code out of the native -bundle. +Story tests are statically extracted CSF `wdio` callbacks. Use `desktopIt` +descriptors when declaring input requirements or endpoint scope. Top-level +functions or named callback collections remain supported for compatibility, +but cannot be selected by an input lane. Callbacks must not close over +module-local values. Import Node-only helpers through literal dynamic +`import()` inside the callback. Keep test code out of the native bundle. ```tsx import type { StoryObj } from '@storybook/react-native'; @@ -37,6 +38,33 @@ export const Default: Story = { }; ``` +For modality-aware tests, import the value helper and use an array of cases: + +```tsx +import { desktopIt, type WdioStory } from '@fluentui-react-native/storybook-desktop/testing'; + +export const Default: WdioStory> = { + args: { testID: 'save-button' }, + wdio: [ + desktopIt('activates in either input lane', { requires: ['click'] }, async ({ browser, expect }) => { + await (await browser.$('~save-button')).click(); + await expect(await browser.$('~saved-message')).toBeDisplayed(); + }), + desktopIt('uses a physical pointer', { requires: ['physical-click'], platforms: ['win32'] }, async ({ browser }) => { + await (await browser.$('~save-button')).click(); + }), + ], +}; +``` + +`requires: ['click']` derives to `both` and adapts `element.click()` to the +selected lane: accessibility click in the accessible lane and physical click +in the physical lane. Explicit physical or accessibility requirements derive +their matching lane. `input` may narrow a `both` case but cannot widen a +transport-specific requirement. `platforms` is an allowlist by default; +`['!win32']` excludes that endpoint. Platform selection uses the target +endpoint, so Win32 remains distinct from Windows. + The callback receives the real WebdriverIO `browser`, `expect-webdriverio` `expect`, typed `desktop` assertions/story commands, target `platform` (`macos`, `windows`, or `win32`), cooperative `signal`, and explicit `skip(reason)`. @@ -73,6 +101,13 @@ editing callbacks because their digest participates in manifest freshness. filters names. `test` operates against an already running app/driver; smoke owns the complete app lifecycle. +Use `storybook test --input accessible|physical` or +`storybook smoke --mode stories-and-tests --input accessible|physical` to +select declared cases. `storybook driver --input ...` supplies the default +lane to a later `storybook test` invocation. `--input` cannot be combined +with a non-default `--click-mode`. A lane requires descriptor cases and rejects +legacy callback collections rather than silently treating them as `both`. + Use the component authoring contract for component-specific test assertions. Legacy `parameters.desktopDriver` static plans remain for compatibility and are not the preferred story-test authoring API. diff --git a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts index 5a199c7cda..ad62fc01bf 100644 --- a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts +++ b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts @@ -48,6 +48,7 @@ import type { DesktopCommandRunner, PreparedDesktopCommand, RunningDesktopComman import { formatDesktopStorybookSmokeTestSummary, runDesktopStorybookSmokeTests } from './smokeTests.js'; import { matchesWdioTest, runWdioStoryTests, selectWdioStories } from '../testing/runWdioTests.js'; import { resolveWdioOptions, type DesktopStorybookWdioOptions } from '../config/wdio.js'; +import { matchesDesktopPlatforms } from '@fluentui-react-native/desktop-driver/authoring'; type ResolvedDesktopStorybookInstance = DesktopStorybookInstance & { driverManifestPath?: string; @@ -92,6 +93,7 @@ export type DesktopStorybookServerOptions = { }; export type DesktopStorybookDriverOptions = DesktopStorybookServerOptions & { + input?: 'accessible' | 'physical'; /** * Protects the driver with a grant authority and publishes one agent's connection descriptor, which the Desktop * Driver MCP adapter reads. WDIO `storybook test` cannot connect to an agent-served driver. @@ -219,6 +221,7 @@ export class DesktopStorybookCli { async driver(platform: Platforms, options: DesktopStorybookDriverOptions = {}): Promise { validateServerOptions(options); validateDriverHost(options.host); + const wdio = resolveWdioOptions({ ...this.config.wdio, ...(options.input ? { input: options.input } : {}) }); const agent = options.agent && { descriptor: options.agent.descriptor ?? defaultAgentDescriptor, // The channel service runs in another working directory. @@ -244,6 +247,7 @@ export class DesktopStorybookCli { resolvedInstance, nativeDriver, agent && { ...agent, ...(agentApplication ? { application: agentApplication } : {}) }, + wdio, ); resolvedInstance.driverManifestPath = driverManifest.path; this.services.start('metro', this.prepareCommand(defaultMetroCommand(resolvedInstance), platform, resolvedInstance)); @@ -329,16 +333,26 @@ export class DesktopStorybookCli { } async test(platform: Platforms, options: DesktopStorybookTestOptions = {}): Promise { - const settings = resolveWdioOptions({ + const overrides: DesktopStorybookWdioOptions = { + ...(options.clickMode === undefined ? {} : { clickMode: options.clickMode }), + ...(options.input === undefined ? {} : { input: options.input }), + ...(options.testReporter === undefined ? {} : { reporter: options.testReporter }), + ...(options.story === undefined ? {} : { story: options.story }), + ...(options.tag === undefined ? {} : { tag: options.tag }), + ...(options.test === undefined ? {} : { test: options.test }), + ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }), + }; + const initialSettings = resolveWdioOptions({ ...this.config.wdio, - ...Object.fromEntries( - Object.entries({ ...options, reporter: options.testReporter }).filter( - ([key, value]) => key !== 'testReporter' && value !== undefined, - ), - ), + ...overrides, }); + const connection = options.list ? undefined : this.testConnection(platform, options); + const settings = + connection?.input && initialSettings.input === undefined + ? resolveWdioOptions({ ...initialSettings, input: connection.input }) + : initialSettings; const manifest = await this.createStoryManifest(this.config, platform); - const selected = selectWdioStories(manifest, settings); + const selected = selectWdioStories(manifest, settings, platform); if (selected.length === 0) { throw new Error('No executable wdio story tests matched the requested selection.'); } @@ -346,13 +360,32 @@ export class DesktopStorybookCli { const value = selected.map(({ id, sourcePath, wdio }) => ({ id, sourcePath, - ...(wdio?.testNames ? { tests: wdio.testNames.filter((name) => matchesWdioTest(name, settings.test)) } : {}), + ...(wdio?.cases + ? { + tests: wdio.cases + .filter( + ({ name, options: descriptor }) => + matchesWdioTest(name, settings.test) && + (!settings.input || descriptor.input === 'both' || descriptor.input === settings.input) && + matchesDesktopPlatforms(descriptor.platforms, platform), + ) + .map(({ name, options: descriptor }) => ({ + name, + input: descriptor.input, + inputSource: descriptor.inputSource, + requires: descriptor.requires ?? [], + platforms: descriptor.platforms, + })), + } + : wdio?.testNames + ? { tests: wdio.testNames.filter((name) => matchesWdioTest(name, settings.test)) } + : {}), })); if (this.reporterMode === 'json' || this.reporterMode === 'ndjson') this.writeStructured(value); else this.output.write(`${JSON.stringify(value, null, 2)}\n`); return; } - const connection = this.testConnection(platform, options); + const activeConnection = connection ?? this.testConnection(platform, options); if (platform === 'macos' && !options.url) { await this.writeMacOSApplicationLease( path.join(this.config.projectRoot, 'storybook-desktop.generated', 'driver-manifest.macos.json'), @@ -362,13 +395,14 @@ export class DesktopStorybookCli { const results = await runWdioStoryTests( { ...settings, - ...connection, + ...activeConnection, config: this.config, manifest, platform, errorOutput: this.errorOutput, report: this.reporter, signal: this.services.controller.signal, + input: settings.input, }, this.runner, ); @@ -378,7 +412,10 @@ export class DesktopStorybookCli { ); } - private testConnection(platform: Platforms, options: DesktopStorybookTestOptions): { targetId: string; url: string } { + private testConnection( + platform: Platforms, + options: DesktopStorybookTestOptions, + ): { input?: 'accessible' | 'physical'; targetId: string; url: string } { if (options.url || options.target) { if (!options.url || !options.target) { throw new Error('Pass --url and --target together, or omit both to use the running Storybook driver instance.'); @@ -415,7 +452,17 @@ export class DesktopStorybookCli { `The Storybook driver instance at ${manifestPath} serves an agent; restart storybook driver --${platform} without --agent to run storybook test.`, ); } - return { targetId: manifest.targetId, url: loopbackUrl(manifest.driverPort) }; + const input = + typeof manifest === 'object' && + manifest !== null && + 'wdio' in manifest && + typeof manifest.wdio === 'object' && + manifest.wdio !== null && + 'input' in manifest.wdio && + (manifest.wdio.input === 'accessible' || manifest.wdio.input === 'physical') + ? manifest.wdio.input + : undefined; + return { targetId: manifest.targetId, url: loopbackUrl(manifest.driverPort), ...(input ? { input } : {}) }; } printInstance(platform: Platforms): void { @@ -478,8 +525,15 @@ export class DesktopStorybookCli { throw unsupportedAction('smoke', platform); } if (smoke?.command) { - const instance = await this.resolveSmokeInstance(platform, smoke, mode); - await this.executePlan(withEnvironment(smoke.command, { [STORYBOOK_SMOKE_MODE]: mode }), platform, instance); + const instance = await this.resolveSmokeInstance(platform, smoke, mode, options.input); + await this.executePlan( + withEnvironment(smoke.command, { + [STORYBOOK_SMOKE_MODE]: mode, + ...(options.input ? { DESKTOP_INPUT: options.input } : {}), + }), + platform, + instance, + ); return; } if (!smoke?.stop) { @@ -488,8 +542,8 @@ export class DesktopStorybookCli { ); } - const instance = await this.resolveSmokeInstance(platform, smoke, mode); - await this.runSmokeLifecycle(platform, smoke, instance, mode); + const instance = await this.resolveSmokeInstance(platform, smoke, mode, options.input); + await this.runSmokeLifecycle(platform, smoke, instance, mode, options.input); } private async executeAction( @@ -509,6 +563,7 @@ export class DesktopStorybookCli { smoke: DesktopSmokeOptions, instance: ResolvedDesktopStorybookInstance, mode: DesktopSmokeMode, + input?: 'accessible' | 'physical', ): Promise { const failures: unknown[] = []; let primaryFailure: unknown; @@ -571,6 +626,7 @@ export class DesktopStorybookCli { targetId: this.targetId(platform), report: this.reporter, signal: this.services.controller.signal, + input, }), ); this.output.write(`${formatDesktopStorybookSmokeTestSummary(result)}\n`); @@ -624,6 +680,7 @@ export class DesktopStorybookCli { platform: Platforms, smoke: DesktopSmokeOptions, mode: DesktopSmokeMode, + input?: 'accessible' | 'physical', ): Promise { const storybookPort = smoke.serverUrl ? portFromUrl(smoke.serverUrl) @@ -641,7 +698,10 @@ export class DesktopStorybookCli { if (mode === 'stories-and-tests') { const nativeDriver = await this.resolveDriver(platform); - resolvedInstance.driverManifestPath = (await this.writeDriverManifest(platform, resolvedInstance, nativeDriver)).path; + const wdio = resolveWdioOptions({ ...this.config.wdio, ...(input ? { input } : {}) }); + resolvedInstance.driverManifestPath = ( + await this.writeDriverManifest(platform, resolvedInstance, nativeDriver, undefined, wdio) + ).path; } if (platform === 'macos') { resolvedInstance.macosXcconfigPath = writeMacOSInstanceConfig(this.config.projectRoot, resolvedInstance); @@ -659,6 +719,7 @@ export class DesktopStorybookCli { instance: DesktopStorybookInstance, nativeDriver: NativeDriverArtifact, agent?: CreateDesktopStorybookDriverManifestOptions['agent'], + wdio?: CreateDesktopStorybookDriverManifestOptions['wdio'], ): Promise<{ manifest: DesktopStorybookDriverManifest; path: string }> { const storyManifest = await this.createStoryManifest(this.config, platform); const outputPath = path.join(this.config.projectRoot, 'storybook-desktop.generated', `driver-manifest.${platform}.json`); @@ -672,6 +733,7 @@ export class DesktopStorybookCli { platform, renderer, storyManifest, + ...(wdio ? { wdio } : {}), }); writeDesktopStorybookDriverManifest(driverManifest, outputPath); return { manifest: driverManifest, path: outputPath }; diff --git a/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts b/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts index 263ab8e307..f7552ff94f 100644 --- a/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts +++ b/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts @@ -36,9 +36,9 @@ type PlatformFlags = { }; type ServerFlags = PlatformFlags & DesktopStorybookServerOptions; -type DriverFlags = ServerFlags & { agent?: boolean; descriptor?: string; root?: string }; +type DriverFlags = ServerFlags & { agent?: boolean; descriptor?: string; root?: string; input?: 'accessible' | 'physical' }; type ManifestFlags = PlatformFlags & { out?: string }; -type SmokeFlags = PlatformFlags & { mode: DesktopSmokeMode }; +type SmokeFlags = PlatformFlags & { mode: DesktopSmokeMode; input?: 'accessible' | 'physical' }; type BuildDriverFlags = PlatformFlags & DesktopStorybookBuildDriverOptions; type PrepFlags = PlatformFlags & { driver: DesktopStorybookPrepOptions['driver'] }; type GetApi = (flags: PlatformFlags) => Promise; @@ -127,14 +127,19 @@ function addTestCommand(program: Command, getApi: GetApi): void { .option('--tag ', 'filter story tags') .option('--url ', 'Desktop Driver URL (requires --target)') .option('--target ', 'registered target ID (requires --url)') + .addOption(new Option('--input ', 'select accessible or physical input tests').choices(['accessible', 'physical'])) .option('--timeout-ms ', 'timeout per executable test', Number) .addOption(new Option('--test-reporter ', 'Node test reporter').choices(['spec', 'tap', 'dot'])) .addOption(new Option('--click-mode ', 'native click semantics').choices(['auto', 'physical', 'accessibility'])); addPlatformOptions(command); command.action(async (flags: PlatformFlags & DesktopStorybookTestOptions) => { + if (flags.input !== undefined && flags.clickMode !== undefined) { + throw new TypeError('--input cannot be combined with --click-mode.'); + } const api = await getApi(flags); await api.test(resolvePlatform(flags, api), { clickMode: flags.clickMode, + input: flags.input, list: flags.list, testReporter: flags.testReporter, story: flags.story, @@ -178,6 +183,7 @@ function addDriverCommand(program: Command, getApi: GetApi): void { .option('--agent', 'require a grant on every driver route and publish one agent connection descriptor') .option('--descriptor ', 'agent connection descriptor record name (default agent-connection.json)') .option('--root ', 'private storage root for the agent descriptor; defaults to the platform location'); + command.addOption(new Option('--input ', 'default input lane for Storybook tests').choices(['accessible', 'physical'])); addPlatformOptions(command); command.action(async (flags: DriverFlags) => { if (!flags.agent && (flags.descriptor !== undefined || flags.root !== undefined)) { @@ -187,6 +193,7 @@ function addDriverCommand(program: Command, getApi: GetApi): void { await api.driver(resolvePlatform(flags, api), { host: flags.host, port: flags.port, + input: flags.input, ...(flags.agent ? { agent: { @@ -387,10 +394,14 @@ function addSmokeCommand(program: Command, getApi: GetApi): void { .choices([...desktopSmokeModes]) .default('stories'), ); + command.addOption(new Option('--input ', 'select accessible or physical input tests').choices(['accessible', 'physical'])); addPlatformOptions(command); command.action(async (flags: SmokeFlags) => { + if (flags.input !== undefined && flags.mode !== 'stories-and-tests') { + throw new TypeError('--input requires --mode stories-and-tests.'); + } const api = await getApi(flags); - await api.smoke(resolvePlatform(flags, api), { mode: flags.mode }); + await api.smoke(resolvePlatform(flags, api), { mode: flags.mode, input: flags.input }); }); } diff --git a/packages/agentic/storybook-desktop/src/cli/smokeTests.ts b/packages/agentic/storybook-desktop/src/cli/smokeTests.ts index 2dcfb9a1f2..afa53f720f 100644 --- a/packages/agentic/storybook-desktop/src/cli/smokeTests.ts +++ b/packages/agentic/storybook-desktop/src/cli/smokeTests.ts @@ -21,6 +21,7 @@ type DesktopStorybookSmokeSession = Pick Promise; export type DesktopStorybookSmokeTestOptions = { + input?: 'accessible' | 'physical'; artifactsRoot?: string; config: DesktopStorybookConfig; driverUrl: string; @@ -52,13 +53,14 @@ export async function runDesktopStorybookSmokeTests( const testTag = options.testTag ?? process.env.STORYBOOK_SMOKE_TAG; const wdioSettings = resolveWdioOptions({ ...config.wdio, + ...(options.input ? { input: options.input } : {}), ...(storyPattern !== undefined ? { story: storyPattern } : {}), ...(testTag !== undefined ? { tag: testTag } : {}), }); const planStoryIds = new Set( selectDesktopStoryTests(manifest, platform, { story: storyPattern, tag: testTag ?? 'desktop-e2e' }).map(({ entry }) => entry.id), ); - const wdioStoryIds = new Set(selectWdioStories(manifest, wdioSettings).map(({ id }) => id)); + const wdioStoryIds = new Set(selectWdioStories(manifest, wdioSettings, platform).map(({ id }) => id)); const storyIds = [...new Set([...planStoryIds, ...wdioStoryIds])].sort(); if (storyIds.length === 0) { throw new Error('No desktop story tests matched the smoke selection.'); diff --git a/packages/agentic/storybook-desktop/src/config/commands.ts b/packages/agentic/storybook-desktop/src/config/commands.ts index 1061ef9350..7df2609e57 100644 --- a/packages/agentic/storybook-desktop/src/config/commands.ts +++ b/packages/agentic/storybook-desktop/src/config/commands.ts @@ -24,6 +24,7 @@ export type DesktopSmokeRunOptions = { * @default "stories" */ mode?: DesktopSmokeMode; + input?: 'accessible' | 'physical'; }; export const STORYBOOK_SMOKE_MODE = 'STORYBOOK_SMOKE_MODE'; diff --git a/packages/agentic/storybook-desktop/src/config/transformStoryTests.test.ts b/packages/agentic/storybook-desktop/src/config/transformStoryTests.test.ts index 9aceedaba9..09d5c21dfb 100644 --- a/packages/agentic/storybook-desktop/src/config/transformStoryTests.test.ts +++ b/packages/agentic/storybook-desktop/src/config/transformStoryTests.test.ts @@ -86,6 +86,16 @@ describe('native Storybook test stripping', () => { expect(output).toContain('Two'); }); + test('strips desktopIt calls and their unused import from the native story module', () => { + const output = transform(` + import { desktopIt, type WdioStory } from '@fluentui-react-native/storybook-desktop/testing'; + export const Example: WdioStory = { render: () => , wdio: [desktopIt('case', () => console.log('node-only'))] }; + `); + expect(output).not.toContain('desktopIt'); + expect(output).not.toContain('node-only'); + expect(output).toContain('render'); + }); + test.each([ [`export const Example = { wdio: importedCallback };`, /inline/], [`export const Example = { ...shared };`, /spread/], diff --git a/packages/agentic/storybook-desktop/src/config/wdio.test.ts b/packages/agentic/storybook-desktop/src/config/wdio.test.ts index dd816c2bfc..996fdf90c4 100644 --- a/packages/agentic/storybook-desktop/src/config/wdio.test.ts +++ b/packages/agentic/storybook-desktop/src/config/wdio.test.ts @@ -24,4 +24,20 @@ describe('Storybook-owned wdio configuration', () => { expect(() => resolveWdioOptions({ story: '' })).toThrow('wdio.story'); expect(() => resolveWdioOptions({ tag: ' ' })).toThrow('wdio.tag'); }); + + test('resolves an explicit input lane and rejects conflicting click configuration', () => { + const previous = process.env.DESKTOP_INPUT; + delete process.env.DESKTOP_INPUT; + try { + expect(resolveWdioOptions({ input: 'accessible' })).toMatchObject({ input: 'accessible', clickMode: 'auto' }); + expect(() => resolveWdioOptions({ input: 'physical', clickMode: 'physical' })).toThrow('cannot be combined with a click mode'); + process.env.DESKTOP_INPUT = 'physical'; + expect(resolveWdioOptions().input).toBe('physical'); + expect(resolveWdioOptions({ input: 'accessible' }).input).toBe('accessible'); + expect(() => resolveWdioOptions({ clickMode: 'physical' })).toThrow('cannot be combined'); + } finally { + if (previous === undefined) delete process.env.DESKTOP_INPUT; + else process.env.DESKTOP_INPUT = previous; + } + }); }); diff --git a/packages/agentic/storybook-desktop/src/config/wdio.ts b/packages/agentic/storybook-desktop/src/config/wdio.ts index 7e7b4102dd..f1525e685e 100644 --- a/packages/agentic/storybook-desktop/src/config/wdio.ts +++ b/packages/agentic/storybook-desktop/src/config/wdio.ts @@ -1,7 +1,9 @@ import type { DesktopWebdriverOptions } from '@fluentui-react-native/desktop-driver/wdio'; +import type { DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; export type DesktopStorybookWdioOptions = { clickMode?: DesktopWebdriverOptions['clickMode']; + input?: DesktopInputLane; reporter?: 'spec' | 'tap' | 'dot'; story?: string; tag?: string; @@ -13,6 +15,15 @@ export function resolveWdioOptions( options: DesktopStorybookWdioOptions = {}, ): Readonly> & DesktopStorybookWdioOptions> { const { timeoutMs = 30_000, reporter = 'spec', clickMode = 'auto' } = options; + const requestedInput = options.input ?? process.env.DESKTOP_INPUT; + let input: DesktopInputLane | undefined; + if (requestedInput !== undefined) { + if (!isDesktopInputLane(requestedInput)) throw new TypeError('wdio.input must be accessible or physical.'); + input = requestedInput; + } + if (input && clickMode !== 'auto') { + throw new TypeError('wdio.input cannot be combined with a click mode other than auto.'); + } if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 2_147_483_647) { throw new TypeError('wdio.timeoutMs must be a positive 32-bit integer.'); } @@ -27,5 +38,15 @@ export function resolveWdioOptions( throw new TypeError(`wdio.${key} must be a non-empty string.`); } } - return Object.freeze({ ...options, timeoutMs, reporter, clickMode }); + return Object.freeze({ + ...options, + ...(input ? { input } : {}), + timeoutMs, + reporter, + clickMode, + }); +} + +function isDesktopInputLane(value: string): value is DesktopInputLane { + return value === 'accessible' || value === 'physical'; } diff --git a/packages/agentic/storybook-desktop/src/driver/driverManifest.ts b/packages/agentic/storybook-desktop/src/driver/driverManifest.ts index 6b5aa6afcc..69dd15764d 100644 --- a/packages/agentic/storybook-desktop/src/driver/driverManifest.ts +++ b/packages/agentic/storybook-desktop/src/driver/driverManifest.ts @@ -59,6 +59,7 @@ export type CreateDesktopStorybookDriverManifestOptions = { platform: Platforms; renderer?: DesktopStorybookDriverManifest['renderer']; storyManifest: DesktopStoryManifest; + wdio?: DesktopStorybookWdioOptions; }; /** Returns the WebDriver target ID the embedded Storybook driver registers for an endpoint and renderer. */ @@ -81,6 +82,7 @@ export function createDesktopStorybookDriverManifest({ platform, renderer = platform === 'win32' ? 'paper' : 'fabric', storyManifest, + wdio, }: CreateDesktopStorybookDriverManifestOptions): DesktopStorybookDriverManifest { const nativeOptions = config.getNativeDriverOptions(platform); if (nativeOptions === false) { @@ -121,7 +123,7 @@ export function createDesktopStorybookDriverManifest({ storybookPort: instance.storybookPort, targetId: desktopStorybookTargetId(config, platform, renderer), testIDPrefix: config.testIDPrefix, - wdio: config.wdio, + wdio: wdio ?? config.wdio, }); } diff --git a/packages/agentic/storybook-desktop/src/driver/storyManifest.ts b/packages/agentic/storybook-desktop/src/driver/storyManifest.ts index c1f0a039f8..837bcf82e2 100644 --- a/packages/agentic/storybook-desktop/src/driver/storyManifest.ts +++ b/packages/agentic/storybook-desktop/src/driver/storyManifest.ts @@ -86,6 +86,7 @@ export async function createDesktopStoryManifest( for (const [exportName, staticStory] of stories) { const wdio = wdioTests.get(exportName); const testNames = wdio?.tests.flatMap(({ name }) => (name === undefined ? [] : [name])); + const cases = wdio?.tests.flatMap(({ name, options }) => (name === undefined || options === undefined ? [] : [{ name, options }])); const tests = readDesktopStoryTests( extractDesktopStoryTests(csf._storyAnnotations?.[exportName]?.parameters, sourceFile, staticStory.id) ?? staticStory.parameters?.desktopDriver, @@ -100,7 +101,16 @@ export async function createDesktopStoryManifest( tags: [...new Set([...(csf.meta?.tags ?? []), ...(staticStory.tags ?? []), 'story'])].sort(), title: csf.meta?.title ?? staticStory.id.split('--')[0], ...(tests ? { tests } : {}), - ...(wdio ? { wdio: { digest: wdio.digest, exportName, ...(testNames?.length ? { testNames } : {}) } } : {}), + ...(wdio + ? { + wdio: { + digest: wdio.digest, + exportName, + ...(testNames?.length ? { testNames } : {}), + ...(cases?.length ? { cases } : {}), + }, + } + : {}), }); } diff --git a/packages/agentic/storybook-desktop/src/testing/desktopIt.ts b/packages/agentic/storybook-desktop/src/testing/desktopIt.ts new file mode 100644 index 0000000000..c95a7580eb --- /dev/null +++ b/packages/agentic/storybook-desktop/src/testing/desktopIt.ts @@ -0,0 +1,11 @@ +import type { WdioDescriptor, WdioStoryCase, WdioStoryTest } from './types.js'; + +export function desktopIt(name: string, run: WdioStoryTest): WdioStoryCase; +export function desktopIt(name: string, options: Omit, run: WdioStoryTest): WdioStoryCase; +export function desktopIt(name: string, optionsOrRun: Omit | WdioStoryTest, run?: WdioStoryTest): WdioStoryCase { + if (!name.trim()) throw new TypeError('A desktop story test requires a non-empty name.'); + const options = typeof optionsOrRun === 'function' ? {} : optionsOrRun; + const callback = typeof optionsOrRun === 'function' ? optionsOrRun : run; + if (typeof callback !== 'function') throw new TypeError('A desktop story test requires a callback.'); + return { name, ...options, run: callback }; +} diff --git a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts index f73abf0e39..0a5ff18bb4 100644 --- a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts +++ b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts @@ -33,6 +33,23 @@ describe('extractWdioTests', () => { ); }); + test('extracts desktopIt calls and their static descriptors without executing the story', () => { + const result = extractWdioTests( + ` + import { desktopIt as test } from '@fluentui-react-native/storybook-desktop/testing'; + export const Example = { wdio: [ + test('accessible action', { requires: ['click'], platforms: ['!win32'] }, async ({ browser }) => { await browser.$('~button').click(); }), + test('default case', async () => {}), + ] }; + `, + sourceFile, + ).get('Example')!; + expect(result.tests.map(({ name, options }) => ({ name, input: options?.input, requires: options?.requires }))).toEqual([ + { name: 'accessible action', input: 'both', requires: ['click'] }, + { name: 'default case', input: 'both', requires: undefined }, + ]); + }); + test.each([ ['{}', /at least one/], ["{ '': () => {} }", /non-empty/], @@ -46,6 +63,20 @@ describe('extractWdioTests', () => { expect(() => extractWdioTests(`export const Example = { wdio: ${value} };`, sourceFile)).toThrow(message); }); + test.each([ + [`export const Example = { wdio: [desktopIt('case', () => {})] };`, /imported from/], + [ + `import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; export const Example = { wdio: [desktopIt('case', { input: mode }, () => {})] };`, + /strict JSON literal/, + ], + [ + `import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; const name = 'case'; export const Example = { wdio: [desktopIt(name, () => {})] };`, + /literal test name/, + ], + ])('rejects non-static desktopIt descriptors', (source, message) => { + expect(() => extractWdioTests(source, sourceFile)).toThrow(message); + }); + test('extracts a standalone typed callback without loading the native story module', async () => { const tests = extractWdioTests( ` diff --git a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts index 493d5239bf..58e2eaf30f 100644 --- a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts +++ b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts @@ -1,6 +1,8 @@ import { createHash } from 'node:crypto'; import { transformSync, type NodePath, type types } from '@babel/core'; +import { validateDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; +import type { ResolvedDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; import storyTests from '../../config/transform-story-tests.cjs'; @@ -8,7 +10,7 @@ const { collectWdioTests, compileWdioTest } = storyTests; export type ExtractedWdioStory = { digest: string; - tests: readonly { name?: string; code: string }[]; + tests: readonly { name?: string; code: string; options?: ResolvedDesktopCaseOptions }[]; }; export function extractWdioTests(code: string, sourceFile: string): ReadonlyMap { @@ -24,9 +26,10 @@ export function extractWdioTests(code: string, sourceFile: string): ReadonlyMap< visitor: { Program(program: NodePath) { for (const [exportName, { callbacks, value }] of collectWdioTests(program)) { - const compiled = callbacks.map(({ callback, name }) => ({ + const compiled = callbacks.map(({ callback, name, options }) => ({ ...(name === undefined ? {} : { name }), code: compileWdioTest(callback, sourceFile, code), + ...(options !== undefined ? { options: validateDesktopCaseOptions(options, `${sourceFile}.wdio`) } : {}), })); const source = code.slice(value.node.start!, value.node.end!); tests.set(exportName, { diff --git a/packages/agentic/storybook-desktop/src/testing/index.ts b/packages/agentic/storybook-desktop/src/testing/index.ts index e5df94e7d3..b1b449f558 100644 --- a/packages/agentic/storybook-desktop/src/testing/index.ts +++ b/packages/agentic/storybook-desktop/src/testing/index.ts @@ -1 +1,2 @@ -export type { WdioStory, WdioStoryContext, WdioStoryTest, WdioStoryTests } from './types.js'; +export { desktopIt } from './desktopIt.js'; +export type { WdioDescriptor, WdioStory, WdioStoryCase, WdioStoryContext, WdioStoryTest, WdioStoryTests } from './types.js'; diff --git a/packages/agentic/storybook-desktop/src/testing/runWdioTests.test.ts b/packages/agentic/storybook-desktop/src/testing/runWdioTests.test.ts new file mode 100644 index 0000000000..5860dd8e4f --- /dev/null +++ b/packages/agentic/storybook-desktop/src/testing/runWdioTests.test.ts @@ -0,0 +1,68 @@ +import { validateDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopStoryManifest } from '@fluentui-react-native/desktop-driver'; + +import { missingRequirements, resolveCaseClickMode, selectWdioCases, selectWdioStories } from './runWdioTests.js'; + +describe('WDIO input lane selection', () => { + const manifest: DesktopStoryManifest = { + endpoint: 'win32', + entries: [ + { + id: 'button', + name: 'Button', + packageName: 'components', + sourcePath: 'button.stories.tsx', + tags: ['story'], + title: 'Button', + wdio: { + digest: 'digest', + exportName: 'Default', + testNames: ['adaptive', 'accessible', 'win32-only'], + cases: [ + { name: 'adaptive', options: validateDesktopCaseOptions({ requires: ['click'] }) }, + { name: 'accessible', options: validateDesktopCaseOptions({ requires: ['accessibility-click'] }) }, + { name: 'win32-only', options: validateDesktopCaseOptions({ platforms: ['win32'] }) }, + ], + }, + }, + ], + platformManifestDigest: 'digest', + portablePlanDigest: 'digest', + schemaVersion: 1, + }; + + test('selects by modality and target platform', () => { + expect(selectWdioStories(manifest, { input: 'accessible' }, 'win32').map(({ id }) => id)).toEqual(['button']); + expect(selectWdioCases(manifest.entries[0], { input: 'accessible' }, 'win32')?.map(({ name }) => name)).toEqual([ + 'adaptive', + 'accessible', + 'win32-only', + ]); + expect(selectWdioCases(manifest.entries[0], {}, 'macos')?.map(({ name }) => name)).toEqual(['adaptive', 'accessible']); + expect(selectWdioCases(manifest.entries[0], { input: 'physical' }, 'win32')?.map(({ name }) => name)).toEqual([ + 'adaptive', + 'win32-only', + ]); + expect(selectWdioCases(manifest.entries[0], { input: 'physical', test: 'accessible' }, 'macos')).toEqual([]); + expect(selectWdioCases(manifest.entries[0], { input: 'physical', test: 'win32-only' }, 'macos')).toEqual([]); + }); + + test('rejects legacy callback entries in a lane', () => { + const legacy = { + ...manifest, + entries: [{ ...manifest.entries[0], wdio: { digest: 'legacy', exportName: 'Default', testNames: ['legacy'] } }], + } satisfies DesktopStoryManifest; + expect(() => selectWdioStories(legacy, { input: 'physical' }, 'win32')).toThrow('declare it with desktopIt'); + }); + + test('adapts click requirements to each lane without fallback', () => { + const descriptor = validateDesktopCaseOptions({ requires: ['click'] }); + const accessibleMode = resolveCaseClickMode(descriptor, 'auto', 'accessible'); + const physicalMode = resolveCaseClickMode(descriptor, 'auto', 'physical'); + expect(accessibleMode).toBe('accessibility'); + expect(physicalMode).toBe('physical'); + expect(missingRequirements(descriptor, { accessibilityClick: true, physicalClick: false }, physicalMode)).toEqual(['click']); + expect(missingRequirements(descriptor, { accessibilityClick: false, physicalClick: true }, accessibleMode)).toEqual(['click']); + expect(missingRequirements(descriptor, { accessibilityClick: true }, accessibleMode)).toEqual([]); + }); +}); diff --git a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts index 52cacc56c1..8ed1c3a371 100644 --- a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts +++ b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts @@ -4,7 +4,8 @@ import path from 'node:path'; import { pathToFileURL } from 'node:url'; import type { DesktopStoryManifest, DesktopStoryManifestEntry } from '@fluentui-react-native/desktop-driver'; -import type { DesktopArtifact } from '@fluentui-react-native/desktop-driver/authoring'; +import { matchesDesktopPlatforms } from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopArtifact, DesktopInputLane, ResolvedDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; import { ArtifactManager, captureDesktopFailureEvidence } from '@fluentui-react-native/desktop-driver/artifacts'; import { connectDesktopWebdriver } from '@fluentui-react-native/desktop-driver/wdio'; import type { DesktopReportWriter } from '@fluentui-react-native/desktop-test/reporting'; @@ -21,10 +22,11 @@ import { formatDesktopStorybookError, writeDesktopStorybookFailure, type Desktop export type WdioStoryTestResult = { storyId: string; testName?: string; - status: 'passed' | 'failed' | 'skipped'; + status: 'passed' | 'failed' | 'skipped' | 'unsupported'; durationMs: number; error?: string; skipReason?: string; + gap?: 'unsupported' | 'quarantine'; evidenceErrors?: string[]; artifacts?: readonly DesktopArtifact[]; }; @@ -53,23 +55,50 @@ export class WdioStoryTestRunError extends Error { } } +class MissingRequirementsError extends Error { + constructor(requirements: readonly string[]) { + super(`Missing desktop requirements: ${requirements.join(', ')}.`); + } +} + export function selectWdioStories( manifest: DesktopStoryManifest, options: DesktopStorybookWdioOptions, + platform?: Platforms, ): readonly DesktopStoryManifestEntry[] { return manifest.entries .filter((entry) => entry.wdio && (!options.story || path.matchesGlob(entry.id, options.story))) .filter((entry) => !options.tag || entry.tags.includes(options.tag)) - .filter((entry) => (entry.wdio?.testNames ?? ['default']).some((name) => matchesWdioTest(name, options.test))) + .filter((entry) => { + if (options.input || entry.wdio?.cases) return (selectWdioCases(entry, options, platform)?.length ?? 0) > 0; + return (entry.wdio?.testNames ?? ['default']).some((name) => matchesWdioTest(name, options.test)); + }) .sort((left, right) => left.id.localeCompare(right.id)); } +export function selectWdioCases( + entry: DesktopStoryManifestEntry, + options: DesktopStorybookWdioOptions, + platform?: Platforms, +): NonNullable['cases'] { + if (!entry.wdio?.cases?.length) { + if (!options.input) return undefined; + throw new Error(`Input lane "${options.input}" cannot run legacy wdio callback in "${entry.id}"; declare it with desktopIt.`); + } + return entry.wdio.cases.filter( + ({ name, options: descriptor }) => + matchesWdioTest(name, options.test) && + (!platform || matchesDesktopPlatforms(descriptor.platforms, platform)) && + (!options.input || descriptor.input === 'both' || descriptor.input === options.input), + ); +} + export async function runWdioStoryTests( options: RunWdioStoryTestsOptions, runner?: DesktopCommandRunner, ): Promise { const settings = resolveWdioOptions(options); - const stories = selectWdioStories(options.manifest, settings); + const stories = selectWdioStories(options.manifest, settings, options.platform); if (stories.length === 0) { throw new Error('No executable wdio story tests matched the requested selection.'); } @@ -79,9 +108,19 @@ export async function runWdioStoryTests( if (!extracted || extracted.digest !== entry.wdio!.digest) { throw new Error(`Executable test source changed for ${entry.id}; regenerate the Storybook manifest.`); } + const selectedCases = selectWdioCases(entry, settings, options.platform); return extracted.tests .filter(({ name }) => matchesWdioTest(name ?? 'default', settings.test)) - .map(({ name, code }) => ({ entry, code, testName: name })); + .map(({ name, code }) => ({ + entry, + code, + testName: name, + descriptor: entry.wdio?.cases?.find((testCase) => testCase.name === name)?.options, + })) + .filter(({ testName }) => { + if (!selectedCases) return !settings.input; + return selectedCases.some(({ name }) => name === testName); + }); }); if (cases.length === 0) throw new Error('No executable wdio tests matched the requested selection.'); const artifacts = new ArtifactManager( @@ -112,7 +151,7 @@ export async function runWdioStoryTests( try { fs.mkdirSync(generatedRoot, { recursive: true }); directory = fs.mkdtempSync(path.join(generatedRoot, 'run-')); - for (const [index, { entry, code, testName }] of cases.entries()) { + for (const [index, { entry, code, testName, descriptor }] of cases.entries()) { options.signal?.throwIfAborted(); const displayName = testName === undefined ? entry.id : `${entry.id} / ${testName}`; const artifactId = @@ -132,9 +171,29 @@ export async function runWdioStoryTests( const context = { story: entry.id, test: testName ?? 'default', endpoint: options.platform }; options.report?.emit({ kind: 'scope-start', payload: { label: displayName, category: 'worker' } }, testId, context, testId); options.report?.emit({ kind: 'test-start', payload: { testId, title: displayName, attempt: 0 } }, testId, context); + const gap = descriptor && declaredGapFor(descriptor, options.platform, settings.input); + if (gap) { + result.status = 'unsupported'; + result.skipReason = gap.reason; + result.gap = gap.kind; + result.durationMs = Date.now() - started; + results.push(result); + options.report?.emit( + { + kind: 'test-result', + payload: { testId, title: displayName, attempt: 0, status: 'unsupported', durationMs: result.durationMs, reason: gap.reason }, + }, + testId, + context, + ); + options.report?.emit({ kind: 'scope-end', payload: { status: 'unsupported' } }, testId); + activeResult = undefined; + continue; + } + const clickMode = resolveCaseClickMode(descriptor, settings.clickMode, settings.input); // The supervisor owns the session, so even a crashed or timed-out Node worker cannot strand it. const desktop = await connectDesktopWebdriver({ - clickMode: settings.clickMode, + clickMode, launchMode: 'attach', platformName: options.platform === 'macos' ? 'macos' : 'windows', targetId: options.targetId, @@ -147,6 +206,8 @@ export async function runWdioStoryTests( if (live.endpoint !== options.platform || live.platformManifestDigest !== options.manifest.platformManifestDigest) { throw new Error('The running Storybook manifest is stale or targets a different platform. Restart storybook driver and the app.'); } + const missing = missingRequirements(descriptor, desktop.browser.capabilities['furn:features'], clickMode); + if (missing.length) throw new MissingRequirementsError(missing); phase = 'navigation'; await desktop.openStory(entry.id); const resultPath = path.join(directory, `${index}.json`); @@ -199,27 +260,32 @@ export async function runWdioStoryTests( result.skipReason = outcome.skipReason; } } catch (error) { - failure = error; - executionFailures.push(error); - result.error = formatDesktopStorybookError(error); - writeDesktopStorybookFailure(`wdio "${displayName}" during ${phase}`, error, options.errorOutput); - const evidence = await captureDesktopFailureEvidence(desktop.session, artifacts, artifactId); - result.artifacts = evidence.artifacts; - result.evidenceErrors = evidence.errors; - for (const artifact of result.artifacts) - options.report?.emit( - { - kind: 'artifact', - payload: { - path: path.relative(options.report.root, path.join(artifacts.root, artifact.path)), - description: `${displayName}: ${artifact.kind}`, + if (error instanceof MissingRequirementsError) { + result.status = 'unsupported'; + result.skipReason = error.message; + } else { + failure = error; + executionFailures.push(error); + result.error = formatDesktopStorybookError(error); + writeDesktopStorybookFailure(`wdio "${displayName}" during ${phase}`, error, options.errorOutput); + const evidence = await captureDesktopFailureEvidence(desktop.session, artifacts, artifactId); + result.artifacts = evidence.artifacts; + result.evidenceErrors = evidence.errors; + for (const artifact of result.artifacts) + options.report?.emit( + { + kind: 'artifact', + payload: { + path: path.relative(options.report.root, path.join(artifacts.root, artifact.path)), + description: `${displayName}: ${artifact.kind}`, + }, }, - }, - testId, - ); - for (const evidenceError of result.evidenceErrors) { - options.report?.fail(new Error(evidenceError), 'evidence', testId); - writeDesktopStorybookFailure(`evidence for "${displayName}"`, evidenceError, options.errorOutput); + testId, + ); + for (const evidenceError of result.evidenceErrors) { + options.report?.fail(new Error(evidenceError), 'evidence', testId); + writeDesktopStorybookFailure(`evidence for "${displayName}"`, evidenceError, options.errorOutput); + } } } phase = 'session cleanup'; @@ -243,7 +309,7 @@ export async function runWdioStoryTests( attempt: 0, status: result.status, durationMs: result.durationMs, - ...(result.status === 'skipped' ? { reason: result.skipReason } : {}), + ...(['skipped', 'unsupported'].includes(result.status) ? { reason: result.skipReason } : {}), ...(failure !== undefined ? { failure: options.report.failure(failure, 'test') } : {}), }, }, @@ -283,7 +349,11 @@ export async function runWdioStoryTests( const failures: unknown[] = infrastructureFailure === undefined ? [] : [infrastructureFailure]; failures.push(...executionFailures); const allSkipped = results.every(({ status }) => status === 'skipped'); - const incomplete = results.length !== cases.length || (allSkipped && !options.optional); + const laneIncomplete = + !!settings.input && + (!results.some(({ status }) => status === 'passed' || status === 'failed') || + results.some(({ status, gap }) => status === 'skipped' || (status === 'unsupported' && gap === undefined))); + const incomplete = results.length !== cases.length || (allSkipped && !options.optional) || laneIncomplete; if (incomplete && failures.length === 0) failures.push(new Error('Required executable story cohort did not execute any tests or left selected work unfinished.')); try { @@ -347,6 +417,75 @@ function csfTestId(storyId: string, testName?: string): string { .digest('hex')}`; } +export function resolveCaseClickMode( + descriptor: ResolvedDesktopCaseOptions | undefined, + configured: 'auto' | 'physical' | 'accessibility', + input: DesktopInputLane | undefined, +): 'auto' | 'physical' | 'accessibility' { + if (input === 'accessible') return 'accessibility'; + if (input === 'physical') return 'physical'; + if (descriptor?.input === 'accessible' || descriptor?.requires?.includes('accessibility-click')) return 'accessibility'; + if (descriptor?.input === 'physical') return 'physical'; + return configured; +} + +export function missingRequirements( + descriptor: ResolvedDesktopCaseOptions | undefined, + features: Readonly> | undefined, + clickMode: 'auto' | 'physical' | 'accessibility', +): string[] { + if (!descriptor?.requires?.length) return []; + const missing: string[] = []; + for (const requirement of descriptor.requires) { + const feature = + requirement === 'click' + ? clickMode === 'physical' + ? 'physicalClick' + : clickMode === 'accessibility' + ? 'accessibilityClick' + : undefined + : requirement === 'physical-click' + ? 'physicalClick' + : requirement === 'accessibility-click' + ? 'accessibilityClick' + : requirement === 'accessibility-actions' + ? 'accessibilityActions' + : requirement === 'element-screenshot' + ? 'elementScreenshot' + : requirement === 'set-window-rect' + ? 'setWindowRect' + : requirement; + if (requirement === 'click' && clickMode === 'auto') { + if (!features?.physicalClick && !features?.accessibilityClick) missing.push(requirement); + } else if (feature && features?.[feature] !== true) { + missing.push(requirement); + } + } + return missing; +} + +function declaredGapFor( + descriptor: ResolvedDesktopCaseOptions, + endpoint: Platforms, + input: DesktopInputLane | undefined, +): { kind: 'unsupported' | 'quarantine'; reason: string } | undefined { + if (descriptor.unsupported && matchesDesktopPlatforms(descriptor.unsupported.platforms, endpoint)) { + return { kind: 'unsupported', reason: descriptor.unsupported.reason }; + } + const quarantine = descriptor.quarantine; + if ( + quarantine && + (!quarantine.platforms || matchesDesktopPlatforms(quarantine.platforms, endpoint)) && + (!quarantine.input || quarantine.input === input) + ) { + if (Date.parse(`${quarantine.expires}T00:00:00Z`) < Date.now()) { + throw new Error(`Quarantine ${quarantine.issue} expired on ${quarantine.expires}.`); + } + return { kind: 'quarantine', reason: `${quarantine.issue}: ${quarantine.reason}` }; + } + return undefined; +} + function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } diff --git a/packages/agentic/storybook-desktop/src/testing/types.ts b/packages/agentic/storybook-desktop/src/testing/types.ts index 8077303d07..d87cbfc6e1 100644 --- a/packages/agentic/storybook-desktop/src/testing/types.ts +++ b/packages/agentic/storybook-desktop/src/testing/types.ts @@ -1,6 +1,7 @@ import type { DesktopWebdriverSession } from '@fluentui-react-native/desktop-driver/wdio'; import type { DesktopEndpoint } from '@fluentui-react-native/desktop-driver'; import type { expect } from 'expect-webdriverio'; +import type { DesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; export type WdioStoryContext = { browser: DesktopWebdriverSession['browser']; @@ -13,8 +14,10 @@ export type WdioStoryContext = { export type WdioStoryTest = (context: WdioStoryContext) => void | Promise; export type WdioStoryTests = Readonly>; +export type WdioDescriptor = DesktopCaseOptions & { run: WdioStoryTest }; +export type WdioStoryCase = WdioDescriptor & { name: string }; /** Wrap StoryObj, importing this type only; callbacks execute in Node, not React Native. */ export type WdioStory = Omit & { - wdio?: WdioStoryTest | WdioStoryTests; + wdio?: readonly WdioStoryCase[] | WdioStoryTest | WdioStoryTests; }; diff --git a/packages/agentic/storybook-desktop/src/testing/wdio.types.test.ts b/packages/agentic/storybook-desktop/src/testing/wdio.types.test.ts index 10a341502e..176a4f80e6 100644 --- a/packages/agentic/storybook-desktop/src/testing/wdio.types.test.ts +++ b/packages/agentic/storybook-desktop/src/testing/wdio.types.test.ts @@ -1,4 +1,4 @@ -import type { WdioStory, WdioStoryContext } from './index.js'; +import { desktopIt, type WdioStory, type WdioStoryContext } from './index.js'; const story: WdioStory = { wdio: async ({ browser, desktop, expect, signal, skip }) => { @@ -38,6 +38,17 @@ const named: WdioStory = { }, }; +const described: WdioStory = { + wdio: [ + desktopIt('uses a lane-adaptive click', { requires: ['click'] }, async ({ browser }) => { + await (await browser.$('~button')).click(); + }), + desktopIt('has the default descriptor', async ({ browser }) => { + await browser.$('~button'); + }), + ], +}; + const wrapped: WdioStory<{ args: { disabled: boolean }; wdio?: unknown }> = { args: { disabled: false }, wdio: async ({ browser }) => { @@ -56,5 +67,6 @@ test('exposes a type-only inline callback contract', () => { expect(typeof story.wdio).toBe('function'); expect(typeof wrapped.wdio).toBe('function'); expect(typeof named.wdio).toBe('object'); + expect(Array.isArray(described.wdio)).toBe(true); expect(invalidArgs.args).toBeDefined(); }); From 451e1cb0225851bccc0b8667be03762f2463871b Mon Sep 17 00:00:00 2001 From: Jason Morse Date: Fri, 6 Nov 2026 17:40:20 -0800 Subject: [PATCH 2/6] Select and account for desktop WDIO cases by input lane Finish part 1 of the input-lane work started in f5a90cdda. Cases declare how they run with desktopIt, and a run selects the cases that apply to its input lane, platform and scope and accounts for everything it excludes. desktop-driver - Add pure case selection and click-mode resolution, and move the requirement preflight and click-mode helpers here. - Add the authoring/input leaf export so config-loading code can import scope helpers from source without the authoring index. desktop-test - Report the lane: a lane run is incomplete when it has skipped or undeclared-unsupported results, an empty selection or nothing executed. - Add the Jasmine adapter (desktop-test/jasmine) with desktopIt and desktopDescribe, plus config-level test scopes and --input. storybook-desktop - Add --input accessible|physical to storybook test, driver and smoke. The lane comes from the flag, DESKTOP_INPUT or the attached driver. - Select cases before they run, resolve declared gaps, assess each cohort, and reject direct input calls in declared cases at extraction. - Fail closed when an accessible lane attaches to a driver that does not mask keyboard, physical click and wheel input. native-core and apps/E2E - Migrate the native-core Callout and FocusZone tests and ButtonV1.spec.win.ts to desktopIt. Update the package references and add patch changesets. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .changeset/input-lanes.md | 3 +- .changeset/native-core-desktop-it.md | 5 + .../src/ButtonV1/specs/ButtonV1.spec.win.ts | 26 +- packages/agentic/desktop-driver/README.md | 35 +- packages/agentic/desktop-driver/package.json | 5 + .../desktop-driver/references/architecture.md | 38 +- .../desktop-driver/references/contributing.md | 35 +- .../references/test-integration.md | 27 +- .../desktop-driver/src/authoring/index.ts | 26 +- .../src/authoring/input.test.ts | 125 +++++- .../desktop-driver/src/authoring/input.ts | 172 +++++++- .../src/authoring/selection.test.ts | 96 +++++ .../desktop-driver/src/authoring/selection.ts | 116 ++++++ packages/agentic/desktop-driver/src/index.ts | 26 +- packages/agentic/desktop-test/package.json | 5 + .../desktop-test/references/commands.md | 35 +- .../desktop-test/references/configuration.md | 28 ++ .../desktop-test/references/contributing.md | 13 +- .../desktop-test/references/reporting.md | 55 ++- .../desktop-test/references/wdio-suites.md | 65 +++ packages/agentic/desktop-test/src/cli.ts | 30 +- packages/agentic/desktop-test/src/config.ts | 67 ++- .../desktop-test/src/contracts.contract.cjs | 236 ++++++++++- .../desktop-test/src/contracts.test.ts | 4 + packages/agentic/desktop-test/src/index.ts | 2 +- .../agentic/desktop-test/src/integration.ts | 3 + .../desktop-test/src/jasmine/adapter.test.ts | 208 ++++++++++ .../desktop-test/src/jasmine/adapter.ts | 103 +++++ .../agentic/desktop-test/src/jasmine/index.ts | 42 ++ .../desktop-test/src/jasmine/registry.ts | 72 ++++ .../src/jasmine/selection.test.ts | 50 +++ .../desktop-test/src/jasmine/selection.ts | 59 +++ .../src/jasmine/specSelection.test.ts | 183 +++++++++ .../desktop-test/src/jasmine/specSelection.ts | 96 +++++ .../src/reporting/collector.test.ts | 121 ++++++ .../desktop-test/src/reporting/collector.ts | 32 +- .../desktop-test/src/reporting/index.ts | 26 +- .../desktop-test/src/reporting/lane.test.ts | 34 ++ .../desktop-test/src/reporting/lane.ts | 37 ++ .../src/reporting/protocol.test.ts | 50 ++- .../desktop-test/src/reporting/protocol.ts | 22 +- .../src/reporting/results.test.ts | 169 +++++++- .../desktop-test/src/reporting/results.ts | 110 ++++- .../src/reporting/runQueries.test.ts | 89 ++++ .../desktop-test/src/reporting/runQueries.ts | 56 ++- .../src/reporting/structured.test.ts | 18 + .../desktop-test/src/reporting/structured.ts | 5 + .../desktop-test/src/supervisor.test.ts | 71 ++++ .../agentic/desktop-test/src/supervisor.ts | 8 + .../desktop-test/src/wdio/configuration.ts | 12 +- .../agentic/desktop-test/src/wdio/reporter.ts | 28 +- packages/agentic/desktop-test/src/wdio/run.ts | 30 +- .../native-core/references/contributing.md | 9 + .../components/callout/Callout.stories.tsx | 383 +++++++++--------- .../callout/menuComposition.wdio.ts | 12 +- .../focus-zone/FocusZone.stories.tsx | 97 ++--- .../src/legacy/callout/Callout.stories.tsx | 75 ++-- .../storybook-desktop/config/server.cjs | 7 + .../config/storybook-control.cjs | 6 +- .../config/transform-story-tests.cjs | 131 ++++++ .../config/transform-story-tests.d.cts | 4 + .../storybook-desktop/references/commands.md | 16 +- .../references/contributing.md | 10 + .../references/story-tests.md | 206 +++++++--- .../src/cli/DesktopStorybookCli.test.ts | 144 ++++++- .../src/cli/DesktopStorybookCli.ts | 83 ++-- .../src/cli/createDesktopStorybookCommand.ts | 18 + .../src/cli/inputLane.test.ts | 101 +++++ .../storybook-desktop/src/cli/inputLane.ts | 42 ++ .../src/cli/smokeTests.test.ts | 124 ++++++ .../storybook-desktop/src/cli/smokeTests.ts | 37 +- .../src/config/integration.ts | 4 +- .../src/config/makeDesktopStorybookConfig.ts | 11 +- .../storybook-desktop/src/config/wdio.test.ts | 5 + .../src/driver/driverManifest.ts | 24 ++ .../src/driver/serverIntegration.test.ts | 58 +++ .../src/testing/extractWdioTests.test.ts | 111 +++++ .../src/testing/extractWdioTests.ts | 19 +- .../src/testing/laneVerdict.test.ts | 75 ++++ .../src/testing/laneVerdict.ts | 72 ++++ .../src/testing/runWdioTests.test.ts | 68 ---- .../src/testing/runWdioTests.ts | 224 ++++------ .../src/testing/selection.test.ts | 301 ++++++++++++++ .../src/testing/selection.ts | 194 +++++++++ .../src/testing/wdioRunner.contract.cjs | 64 +++ .../src/testing/wdioRunner.test.ts | 25 ++ 86 files changed, 5007 insertions(+), 762 deletions(-) create mode 100644 .changeset/native-core-desktop-it.md create mode 100644 packages/agentic/desktop-driver/src/authoring/selection.test.ts create mode 100644 packages/agentic/desktop-driver/src/authoring/selection.ts create mode 100644 packages/agentic/desktop-test/src/jasmine/adapter.test.ts create mode 100644 packages/agentic/desktop-test/src/jasmine/adapter.ts create mode 100644 packages/agentic/desktop-test/src/jasmine/index.ts create mode 100644 packages/agentic/desktop-test/src/jasmine/registry.ts create mode 100644 packages/agentic/desktop-test/src/jasmine/selection.test.ts create mode 100644 packages/agentic/desktop-test/src/jasmine/selection.ts create mode 100644 packages/agentic/desktop-test/src/jasmine/specSelection.test.ts create mode 100644 packages/agentic/desktop-test/src/jasmine/specSelection.ts create mode 100644 packages/agentic/desktop-test/src/reporting/lane.test.ts create mode 100644 packages/agentic/desktop-test/src/reporting/lane.ts create mode 100644 packages/agentic/desktop-test/src/supervisor.test.ts create mode 100644 packages/agentic/storybook-desktop/src/cli/inputLane.test.ts create mode 100644 packages/agentic/storybook-desktop/src/cli/inputLane.ts create mode 100644 packages/agentic/storybook-desktop/src/testing/laneVerdict.test.ts create mode 100644 packages/agentic/storybook-desktop/src/testing/laneVerdict.ts delete mode 100644 packages/agentic/storybook-desktop/src/testing/runWdioTests.test.ts create mode 100644 packages/agentic/storybook-desktop/src/testing/selection.test.ts create mode 100644 packages/agentic/storybook-desktop/src/testing/selection.ts diff --git a/.changeset/input-lanes.md b/.changeset/input-lanes.md index 4c06b6b268..0b98bd6d1c 100644 --- a/.changeset/input-lanes.md +++ b/.changeset/input-lanes.md @@ -1,6 +1,7 @@ --- '@fluentui-react-native/desktop-driver': patch +'@fluentui-react-native/desktop-test': patch '@fluentui-react-native/storybook-desktop': patch --- -Add modality-aware desktop test descriptors and Storybook input-lane selection. +Add modality-aware desktop test descriptors: `desktopIt` and `desktopDescribe` declare `requires`, `input`, `platforms`, `scope`, and known gaps for Storybook cases and, through `desktop-test/jasmine`, WebdriverIO Jasmine specs. `--input accessible|physical` (or `DESKTOP_INPUT`) runs one input lane, selects the cases that apply, masks the host for the accessible lane, and reports the lane and what it excluded. diff --git a/.changeset/native-core-desktop-it.md b/.changeset/native-core-desktop-it.md new file mode 100644 index 0000000000..9195e49344 --- /dev/null +++ b/.changeset/native-core-desktop-it.md @@ -0,0 +1,5 @@ +--- +'@fluentui-react-native/native-core': patch +--- + +Declare the Callout and FocusZone desktop tests with `desktopIt` and `requires`, so each runs in the input lane it needs and the Win32 Tab-exit gap is declared instead of skipped at run time. diff --git a/apps/E2E/src/ButtonV1/specs/ButtonV1.spec.win.ts b/apps/E2E/src/ButtonV1/specs/ButtonV1.spec.win.ts index 2d03c4cb2d..4542acf99f 100644 --- a/apps/E2E/src/ButtonV1/specs/ButtonV1.spec.win.ts +++ b/apps/E2E/src/ButtonV1/specs/ButtonV1.spec.win.ts @@ -1,14 +1,17 @@ +import { desktopDescribe, desktopIt } from '@fluentui-react-native/desktop-test/jasmine'; + import { BUTTON_ACCESSIBILITY_LABEL, BUTTON_TEST_COMPONENT_LABEL } from '../../ButtonLegacy/consts'; import { BUTTON_A11Y_ROLE, Keys, Attribute } from '../../common/consts'; import ButtonV1PageObject from '../pages/ButtonV1PageObject'; // Before testing begins, allow up to 60 seconds for app to open -describe('ButtonV1 Testing Initialization', () => { - it('Wait for app load', async () => { +desktopDescribe('ButtonV1 Testing Initialization', () => { + desktopIt('Wait for app load', async () => { expect(await ButtonV1PageObject.waitForInitialPageToDisplay()).toBeTrue(); }); - it('Click and navigate to ButtonV1 test page', async () => { + // Navigation scrolls the page list with the wheel whenever the page button is out of view. + desktopIt('Click and navigate to ButtonV1 test page', { requires: ['click', 'wheel'] }, async () => { /* Click on component button to navigate to test page */ expect(await ButtonV1PageObject.navigateToPageAndLoadTests()).toBeTrue(); @@ -21,18 +24,19 @@ describe('ButtonV1 Testing Initialization', () => { }); }); -describe('ButtonV1 Accessibility Testing', () => { +// The setup scrolls the test element into view, which can inject native wheel input, so this group is physical. +desktopDescribe('ButtonV1 Accessibility Testing', { requires: ['wheel'] }, () => { beforeEach(async () => { await ButtonV1PageObject.scrollToTestElement(); }); - it('Validate "accessibilityRole" value defaults to "ControlType.Button".', async () => { + desktopIt('Validate "accessibilityRole" value defaults to "ControlType.Button".', async () => { await expect( await ButtonV1PageObject.compareAttribute(ButtonV1PageObject._primaryComponent, Attribute.AccessibilityRole, BUTTON_A11Y_ROLE), ).toBeTruthy(); }); - it('Set "accessibilityLabel" prop. Validate "accessibilityLabel" value propagates to "Name" element attribute.', async () => { + desktopIt('Set "accessibilityLabel" prop. Validate "accessibilityLabel" value propagates to "Name" element attribute.', async () => { await expect( await ButtonV1PageObject.compareAttribute( ButtonV1PageObject._primaryComponent, @@ -42,7 +46,7 @@ describe('ButtonV1 Accessibility Testing', () => { ).toBeTruthy(); }); - it('Do NOT set "accessibilityLabel" prop. Validate "Name" element attribute defaults to the button label.', async () => { + desktopIt('Do NOT set "accessibilityLabel" prop. Validate "Name" element attribute defaults to the button label.', async () => { await expect( await ButtonV1PageObject.compareAttribute( ButtonV1PageObject._secondaryComponent, @@ -53,7 +57,7 @@ describe('ButtonV1 Accessibility Testing', () => { }); }); -describe('ButtonV1 Functional Testing', () => { +desktopDescribe('ButtonV1 Functional Testing', { requires: ['wheel'] }, () => { /* Scrolls and waits for the Button to be visible on the Test Page */ beforeEach(async () => { await ButtonV1PageObject.scrollToTestElement(); @@ -63,7 +67,7 @@ describe('ButtonV1 Functional Testing', () => { await ButtonV1PageObject.resetTest(); }); - it('Click primary button. Validate onClick() callback was fired.', async () => { + desktopIt('Click primary button. Validate onClick() callback was fired.', { requires: ['physical-click'] }, async () => { await ButtonV1PageObject.click(ButtonV1PageObject._primaryComponent); await expect( await ButtonV1PageObject.waitForOnClickCallbackToFire(`The primary button failed to fire an onClick callback with a mouse click.`), @@ -74,7 +78,7 @@ describe('ButtonV1 Functional Testing', () => { .toBeFalsy(); }); - it('Type "Enter" on primary button. Validate onClick() callback was fired.', async () => { + desktopIt('Type "Enter" on primary button. Validate onClick() callback was fired.', { requires: ['keyboard'] }, async () => { await ButtonV1PageObject.sendKeys(ButtonV1PageObject._primaryComponent, [Keys.ENTER]); await expect( await ButtonV1PageObject.waitForOnClickCallbackToFire( @@ -87,7 +91,7 @@ describe('ButtonV1 Functional Testing', () => { .toBeFalsy(); }); - it('Type "Space" on primary button. Validate onClick() callback was fired.', async () => { + desktopIt('Type "Space" on primary button. Validate onClick() callback was fired.', { requires: ['keyboard'] }, async () => { await ButtonV1PageObject.sendKeys(ButtonV1PageObject._primaryComponent, [Keys.SPACE]); await expect( diff --git a/packages/agentic/desktop-driver/README.md b/packages/agentic/desktop-driver/README.md index e061eb895a..484c27638b 100644 --- a/packages/agentic/desktop-driver/README.md +++ b/packages/agentic/desktop-driver/README.md @@ -139,23 +139,24 @@ The [reference index](references/README.md) describes each package-owned guide. ## Public entry points -| Export | Purpose | -| --------------- | ----------------------------------------------------------------------------------------- | -| Package root | Native-helper build/resolution, `DesktopHost`, server, low-level client, and common types | -| `/authoring` | Serializable story plans, selectors, capabilities, and result types | -| `/wdio` | WebdriverIO connection, custom commands, assertions, and story-plan runner | -| `/client` | Low-level typed W3C client | -| `/authority` | Grants, private connection descriptors, and macOS owner-validated storage | -| `/native` | Native-helper build, resolution, and verification without loading the package root | -| `/server` | Embeddable remote end, session manager, and target registry | -| `/agent` | Bounded JSON-safe inspection, action, and evidence API | -| `/operations` | Versioned agent operation catalog, executor, and result types | -| `/artifacts` | Confined, atomic evidence persistence | -| `/runner` | Plan selection, sharding, execution, and result classification | -| `/storybook` | Story manifest and orchestration contracts, without a Storybook dependency | -| `/testing` | Fake host, fake Storybook orchestrator, and protocol harnesses | -| `/cli` | Embeddable Commander command and CLI runner | -| `/cli/reporter` | Shared reporter mode resolution and structured output framing | +| Export | Purpose | +| ------------------ | ----------------------------------------------------------------------------------------- | +| Package root | Native-helper build/resolution, `DesktopHost`, server, low-level client, and common types | +| `/authoring` | Serializable story plans, selectors, capabilities, result types, and case descriptors | +| `/wdio` | WebdriverIO connection, custom commands, assertions, and story-plan runner | +| `/client` | Low-level typed W3C client | +| `/authority` | Grants, private connection descriptors, and macOS owner-validated storage | +| `/native` | Native-helper build, resolution, and verification without loading the package root | +| `/server` | Embeddable remote end, session manager, and target registry | +| `/agent` | Bounded JSON-safe inspection, action, and evidence API | +| `/operations` | Versioned agent operation catalog, executor, and result types | +| `/artifacts` | Confined, atomic evidence persistence | +| `/runner` | Plan selection, sharding, execution, and result classification | +| `/storybook` | Story manifest and orchestration contracts, without a Storybook dependency | +| `/testing` | Fake host, fake Storybook orchestrator, and protocol harnesses | +| `/cli` | Embeddable Commander command and CLI runner | +| `/cli/reporter` | Shared reporter mode resolution and structured output framing | +| `/authoring/input` | Descriptor vocabulary and scope rules with no runtime imports (loads from source) | The service implements native application semantics rather than pretending to be a browser. Unsupported browser features return `unsupported operation` diff --git a/packages/agentic/desktop-driver/package.json b/packages/agentic/desktop-driver/package.json index 9764867493..bb7b57efae 100644 --- a/packages/agentic/desktop-driver/package.json +++ b/packages/agentic/desktop-driver/package.json @@ -53,6 +53,11 @@ "import": "./lib/authoring/index.js", "default": "./src/authoring/index.ts" }, + "./authoring/input": { + "types": "./lib/authoring/input.d.ts", + "import": "./lib/authoring/input.js", + "default": "./src/authoring/input.ts" + }, "./agent": { "types": "./lib/agent/index.d.ts", "import": "./lib/agent/index.js", diff --git a/packages/agentic/desktop-driver/references/architecture.md b/packages/agentic/desktop-driver/references/architecture.md index 13a181c733..ae89c8e124 100644 --- a/packages/agentic/desktop-driver/references/architecture.md +++ b/packages/agentic/desktop-driver/references/architecture.md @@ -120,25 +120,25 @@ handoff are still work in progress; see [current status](current-status.md). ## Module ownership -| Module | Responsibility | -| ------------------ | ------------------------------------------------------------------ | -| `src/protocol` | W3C parsing, actions, capabilities, timeouts, and errors | -| `src/server` | HTTP routing, target registry, sessions, windows, and elements | -| `src/host` | Platform-neutral `DesktopHost` and target contracts | -| `src/hosts/fake` | Deterministic in-memory provider | -| `src/hosts/native` | Long-lived helper process and `DesktopHost` adapter | -| `src/native` | Build, cache, resolution, verification, manifests, and wire types | -| `native/macos` | Swift operating-system implementation | -| `native/windows` | Shared C++ Windows and Win32 implementation | -| `src/client` | Low-level typed W3C client | -| `src/wdio` | Sanctioned WebdriverIO integration and custom commands | -| `src/authoring` | Serializable story plans, selectors, capabilities, and results | -| `src/runner` | Filtering, sharding, execution, and result classification | -| `src/artifacts` | Confined atomic evidence | -| `src/agent` | Bounded JSON-safe operations | -| `src/operations` | Agent operation catalog, executor, references, ledger, and journal | -| `src/testing` | Fake harnesses and Storybook test doubles | -| `src/cli` | Commander parsing, JSON output, and process exit adaptation | +| Module | Responsibility | +| ------------------ | -------------------------------------------------------------------------------- | +| `src/protocol` | W3C parsing, actions, capabilities, timeouts, and errors | +| `src/server` | HTTP routing, target registry, sessions, windows, and elements | +| `src/host` | Platform-neutral `DesktopHost` and target contracts | +| `src/hosts/fake` | Deterministic in-memory provider | +| `src/hosts/native` | Long-lived helper process and `DesktopHost` adapter | +| `src/native` | Build, cache, resolution, verification, manifests, and wire types | +| `native/macos` | Swift operating-system implementation | +| `native/windows` | Shared C++ Windows and Win32 implementation | +| `src/client` | Low-level typed W3C client | +| `src/wdio` | Sanctioned WebdriverIO integration and custom commands | +| `src/authoring` | Serializable story plans, selectors, capabilities, results, and case descriptors | +| `src/runner` | Filtering, sharding, execution, and result classification | +| `src/artifacts` | Confined atomic evidence | +| `src/agent` | Bounded JSON-safe operations | +| `src/operations` | Agent operation catalog, executor, references, ledger, and journal | +| `src/testing` | Fake harnesses and Storybook test doubles | +| `src/cli` | Commander parsing, JSON output, and process exit adaptation | ## Deliberate boundaries diff --git a/packages/agentic/desktop-driver/references/contributing.md b/packages/agentic/desktop-driver/references/contributing.md index 40e9c15a12..9d519cef3c 100644 --- a/packages/agentic/desktop-driver/references/contributing.md +++ b/packages/agentic/desktop-driver/references/contributing.md @@ -23,7 +23,7 @@ change-oriented guidance. | `src/native` | Source build, cache, selection, verification, and native artifact types | | `native/macos` | Swift implementation and self-test | | `native/windows` | Shared C++ Windows/Win32 implementation and self-test | -| `src/authoring` | Static JSON plan and result schemas | +| `src/authoring` | Static JSON plan and result schemas, and the case descriptor | | `src/runner` | Plan selection, capability checks, execution, and classification | | `src/wdio` | Sanctioned WebdriverIO connection and commands | | `src/client` | Low-level typed W3C client | @@ -38,6 +38,39 @@ Do not duplicate native build, cache, or verification logic in Storybook consumers. Do not move Storybook manifest generation or app lifecycle into this package. +## Case descriptors and input lanes + +`src/authoring/input.ts` and `src/authoring/selection.ts` are the one +implementation of the test case descriptor (`requires`, `input`, `platforms`, +`scope`, and declared gaps). Storybook extraction and its runner, and the +desktop-test Jasmine adapter, all call them; do not reimplement validation, +merging, or a decision in a consumer. Keep both files pure data and functions, +with no Node, React, or WebdriverIO imports, because stories and workers load +them. `input.ts` also has no runtime imports at all (its one import is a type), +and the `./authoring/input` subpath exports it directly: desktop-test's config +loader reads scope domains from it, and a config file is loaded from source +through a CommonJS fallback that cannot resolve this package's `.js` +specifiers, so keep it that way. + +- Add a requirement to `desktopRequirements`, to the physical or accessible + requirement set in `input.ts` when it implies an input, and to the + `requirementFeature` map in `selection.ts` for the host feature that backs it, + together. A requirement whose feature a host did not report is unmet. +- `mergeDesktopCaseOptions` only narrows: requirements union, `platforms` and + `scope` intersect, and a nested declaration can never widen or change its + group's input. An overlap that is empty is an error because the case could + never run. A group cannot carry a gap. +- `decideDesktopCase` judges platform, then scope, then the lane, so a case is + excluded for one reason. It resolves a declared gap too, so an expired + quarantine stops a run before anything executes. +- `resolveDesktopClickMode` lets a lane decide the click mode outright. That is + how one `click` requirement is an accessibility click in the accessible lane + and a physical click in the physical lane. +- `accessibleLaneMaskedFeatures` (`keyboard`, `physicalClick`, `wheel`) is what + an accessible lane disables through the native host's `disabledInputFeatures`, + and `unmaskedAccessibleFeatures` fails closed when a host reports any of them + as still available. Change the set together with the hosts that apply it. + ## Behavioral rules - Reserve a target before asynchronous launch or attach. diff --git a/packages/agentic/desktop-driver/references/test-integration.md b/packages/agentic/desktop-driver/references/test-integration.md index 82e57e60be..6e09fb1c46 100644 --- a/packages/agentic/desktop-driver/references/test-integration.md +++ b/packages/agentic/desktop-driver/references/test-integration.md @@ -98,15 +98,24 @@ exits, including on assertion failures, timeouts, or crashes. WebdriverIO's sets `waitforInterval` (100 ms) and `waitforTimeout` (5 s) explicitly; without an interval, `waitUntil` and `waitFor*` stop after their first unsatisfied poll. -Storybook's preferred top-level `wdio` functions and named function collections -use this pattern with Node's built-in test runner. Each function receives the -target `platform` (`macos`, `windows`, or `win32`), and each named case gets an -independent worker/session. Native feature flags are typed on the WebdriverIO -browser's `furn:features` capability. Button now uses this authoring pattern -instead of serializable plans. Extraction, native-bundle stripping, runner -configuration, and source freshness belong to -`@fluentui-react-native/storybook-desktop`, -not this Storybook-independent package. Static plans below are unchanged. +Storybook's preferred top-level `wdio` export is an array of `desktopIt` cases, +which declare the input and platforms they need. Its older functions and named +function collections use the same pattern with Node's built-in test runner. +Each function receives the target `platform` (`macos`, `windows`, or `win32`), +and each named case gets an independent worker/session. Native feature flags are +typed on the WebdriverIO browser's `furn:features` capability. Button now uses +this authoring pattern instead of serializable plans. Extraction, native-bundle +stripping, runner configuration, and source freshness belong to +`@fluentui-react-native/storybook-desktop`, not this Storybook-independent +package. Static plans below are unchanged. + +The descriptor that `desktopIt` accepts (`requires`, `input`, `platforms`, +`scope`, and declared gaps) is defined and decided in `./authoring`, so Storybook +and `desktop-test` share one implementation. A runner that executes one input lane +pins `furn:clickMode` with `resolveDesktopClickMode`: `element.click()` is then an +accessibility click in the accessible lane and a physical click in the physical +lane. `missingDesktopRequirements` judges a case's `requires` against the +session's `furn:features`, and judges a `click` requirement by the pinned mode. ## Author portable story plans diff --git a/packages/agentic/desktop-driver/src/authoring/index.ts b/packages/agentic/desktop-driver/src/authoring/index.ts index 0fd5cd6695..96cae618c8 100644 --- a/packages/agentic/desktop-driver/src/authoring/index.ts +++ b/packages/agentic/desktop-driver/src/authoring/index.ts @@ -14,15 +14,39 @@ export { desktopStoryPlatforms, validateDesktopStoryTests, } from './storyTests.js'; -export { desktopRequirements, matchesDesktopPlatforms, resolveDesktopInput, validateDesktopCaseOptions } from './input.js'; +export { + accessibleLaneMaskedFeatures, + desktopInputLanes, + desktopRequirements, + matchesDesktopPlatforms, + matchesDesktopScope, + mergeDesktopCaseOptions, + resolveDesktopInput, + resolveDesktopScopeDomains, + resolveDesktopScopeValues, + unmaskedAccessibleFeatures, + validateDesktopCaseOptions, + validateDesktopGroupOptions, +} from './input.js'; export type { + AccessibleLaneMaskedFeature, DesktopCaseOptions, DesktopInput, DesktopInputLane, DesktopPlatforms, DesktopRequirement, + DesktopScopeDomains, + DesktopScopeValues, ResolvedDesktopCaseOptions, } from './input.js'; +export { decideDesktopCase, missingDesktopRequirements, resolveDesktopClickMode } from './selection.js'; +export type { + DesktopCaseClickMode, + DesktopCaseDecision, + DesktopCaseTarget, + DesktopDeclaredGap, + DesktopExclusionReason, +} from './selection.js'; export type { DesktopStoryCapability, DesktopStoryExpectation, diff --git a/packages/agentic/desktop-driver/src/authoring/input.test.ts b/packages/agentic/desktop-driver/src/authoring/input.test.ts index 1c02abba19..0077d51cfc 100644 --- a/packages/agentic/desktop-driver/src/authoring/input.test.ts +++ b/packages/agentic/desktop-driver/src/authoring/input.test.ts @@ -1,4 +1,88 @@ -import { matchesDesktopPlatforms, resolveDesktopInput, validateDesktopCaseOptions } from './input.js'; +import { + accessibleLaneMaskedFeatures, + matchesDesktopPlatforms, + matchesDesktopScope, + mergeDesktopCaseOptions, + resolveDesktopInput, + resolveDesktopScopeDomains, + resolveDesktopScopeValues, + unmaskedAccessibleFeatures, + validateDesktopCaseOptions, + validateDesktopGroupOptions, +} from './input.js'; + +describe('accessible lane masks', () => { + test('lists the features a lane must hide, and fails closed when a host does not report them', () => { + expect(accessibleLaneMaskedFeatures).toEqual(['keyboard', 'physicalClick', 'wheel']); + expect(unmaskedAccessibleFeatures({ keyboard: false, physicalClick: false, wheel: false, focus: true })).toEqual([]); + expect(unmaskedAccessibleFeatures({ keyboard: true, physicalClick: false, wheel: false })).toEqual(['keyboard']); + expect(unmaskedAccessibleFeatures({ keyboard: false })).toEqual(['physicalClick', 'wheel']); + expect(unmaskedAccessibleFeatures(undefined)).toEqual(['keyboard', 'physicalClick', 'wheel']); + expect(unmaskedAccessibleFeatures('masked')).toEqual(['keyboard', 'physicalClick', 'wheel']); + }); +}); + +describe('desktop option groups', () => { + const group = (options: object) => validateDesktopGroupOptions(options); + const kase = (options: object) => validateDesktopCaseOptions(options); + + test('a group cannot declare a gap, because a gap describes one case', () => { + expect(() => group({ unsupported: { platforms: ['win32'], reason: 'No.' } })).toThrow('declare the gap on the case'); + expect(() => group({ quarantine: { issue: '#1', reason: 'Flaky.', expires: '2099-01-01' } })).toThrow('declare the gap on the case'); + }); + + test('composes requirements, platforms and scope, and derives the input from the union', () => { + const merged = mergeDesktopCaseOptions( + group({ requires: ['wheel'], platforms: ['!win32'], scope: { 'rn.renderer': ['fabric', 'paper'] } }), + kase({ requires: ['wheel', 'focus'], platforms: ['windows', 'win32'], scope: { 'rn.renderer': ['fabric'] } }), + ); + expect(merged).toEqual({ + requires: ['wheel', 'focus'], + platforms: ['windows'], + scope: { 'rn.renderer': ['fabric'] }, + input: 'physical', + inputSource: 'derived', + }); + }); + + test('returns the case unchanged outside any group', () => { + const options = kase({ requires: ['click'] }); + expect(mergeDesktopCaseOptions(undefined, options)).toBe(options); + }); + + test('lets a group narrow the input, but a nested declaration can never widen or switch it', () => { + const physical = group({ input: 'physical' }); + expect(mergeDesktopCaseOptions(physical, kase({}))).toMatchObject({ input: 'physical', inputSource: 'declared' }); + expect(mergeDesktopCaseOptions(physical, kase({ input: 'physical' }))).toMatchObject({ input: 'physical' }); + expect(() => mergeDesktopCaseOptions(physical, kase({ input: 'both' }))).toThrow('cannot change the enclosing group'); + expect(() => mergeDesktopCaseOptions(physical, kase({ input: 'accessible' }))).toThrow('cannot change the enclosing group'); + expect(() => mergeDesktopCaseOptions(physical, kase({ requires: ['accessibility-click'] }))).toThrow('cannot widen'); + }); + + test('keeps a group that requires a transport from being widened or mixed by a case', () => { + const keyboard = group({ requires: ['keyboard'] }); + expect(() => mergeDesktopCaseOptions(keyboard, kase({ input: 'both' }))).toThrow('cannot widen'); + expect(() => mergeDesktopCaseOptions(keyboard, kase({ requires: ['accessibility-click'] }))).toThrow('split the case'); + }); + + test('an explicit both on a group narrows nothing', () => { + const open = group({ input: 'both' }); + expect(mergeDesktopCaseOptions(open, kase({ requires: ['keyboard'] }))).toMatchObject({ input: 'physical', inputSource: 'derived' }); + expect(mergeDesktopCaseOptions(open, kase({}))).toMatchObject({ input: 'both', inputSource: 'derived' }); + }); + + test('rejects a case that could never run', () => { + expect(() => mergeDesktopCaseOptions(group({ platforms: ['win32'] }), kase({ platforms: ['!win32'] }))).toThrow('could never run'); + expect(() => + mergeDesktopCaseOptions(group({ scope: { 'rn.renderer': ['fabric'] } }), kase({ scope: { 'rn.renderer': ['paper'] } })), + ).toThrow('could never run'); + }); + + test('carries a case gap through its groups', () => { + const unsupported = { platforms: ['win32'], reason: 'Win32 keeps the cycle behavior.', issue: '#4377' } as const; + expect(mergeDesktopCaseOptions(group({ requires: ['keyboard'] }), kase({ unsupported }))).toMatchObject({ unsupported }); + }); +}); describe('desktop input descriptors', () => { test.each([ @@ -30,3 +114,42 @@ describe('desktop input descriptors', () => { expect(() => validateDesktopCaseOptions({ requires: ['input'] })).toThrow('unsupported requirement'); }); }); + +describe('desktop scope domains', () => { + const domains = resolveDesktopScopeDomains({ flavor: ['stable', 'insider'] }); + const supplied = resolveDesktopScopeValues({ 'rn.renderer': 'fabric', flavor: 'stable' }, domains); + + test('merges custom domains with the built-in renderer and reserves rn.*', () => { + expect(domains).toEqual({ 'rn.renderer': ['fabric', 'paper'], flavor: ['stable', 'insider'] }); + expect(resolveDesktopScopeDomains()).toEqual({ 'rn.renderer': ['fabric', 'paper'] }); + expect(() => resolveDesktopScopeDomains({ 'rn.renderer': ['x'] })).toThrow('reserved'); + expect(() => resolveDesktopScopeDomains({ 'rn.other': ['x'] })).toThrow('reserved'); + expect(() => resolveDesktopScopeDomains({ Flavor: ['x'] })).toThrow('invalid key'); + expect(() => resolveDesktopScopeDomains({ flavor: [] })).toThrow('non-empty'); + }); + + test('rejects supplied values outside their domain', () => { + expect(() => resolveDesktopScopeValues({ unknown: 'a' }, domains)).toThrow('unknown scope key'); + expect(() => resolveDesktopScopeValues({ 'rn.renderer': 'metal' }, domains)).toThrow('must be one of: fabric, paper'); + expect(() => resolveDesktopScopeValues({ flavor: 3 }, domains)).toThrow('must be one of'); + expect(() => resolveDesktopScopeValues({ constructor: 'a' }, domains)).toThrow('unknown scope key'); + }); + + test('ORs values within a key and ANDs keys', () => { + expect(matchesDesktopScope(undefined, supplied, domains)).toBe(true); + expect(matchesDesktopScope({ 'rn.renderer': ['fabric', 'paper'] }, supplied, domains)).toBe(true); + expect(matchesDesktopScope({ 'rn.renderer': ['paper'] }, supplied, domains)).toBe(false); + expect(matchesDesktopScope({ 'rn.renderer': ['fabric'], flavor: ['stable'] }, supplied, domains)).toBe(true); + expect(matchesDesktopScope({ 'rn.renderer': ['fabric'], flavor: ['insider'] }, supplied, domains)).toBe(false); + }); + + test('treats an undeclared key or value, or an unsupplied key, as a configuration error even when another key excludes', () => { + expect(() => matchesDesktopScope({ 'rn.renderer': ['paper'], missing: ['a'] }, supplied, domains)).toThrow( + 'unknown scope key "missing"', + ); + expect(() => matchesDesktopScope({ 'rn.renderer': ['paper'], flavor: ['beta'] }, supplied, domains)).toThrow('"beta"'); + expect(() => matchesDesktopScope({ flavor: ['stable'] }, resolveDesktopScopeValues({}, domains), domains)).toThrow('not supplied'); + const inherited: Record = JSON.parse('{"constructor":["a"]}'); + expect(() => matchesDesktopScope(inherited, supplied, domains)).toThrow('unknown scope key'); + }); +}); diff --git a/packages/agentic/desktop-driver/src/authoring/input.ts b/packages/agentic/desktop-driver/src/authoring/input.ts index ee0e353c51..b81575801e 100644 --- a/packages/agentic/desktop-driver/src/authoring/input.ts +++ b/packages/agentic/desktop-driver/src/authoring/input.ts @@ -16,6 +16,21 @@ export const desktopRequirements = [ export type DesktopRequirement = (typeof desktopRequirements)[number]; export type DesktopInput = 'accessible' | 'physical' | 'both'; export type DesktopInputLane = Exclude; +export const desktopInputLanes = ['accessible', 'physical'] as const satisfies readonly DesktopInputLane[]; + +/** + * The input an accessible lane keeps from reaching the application. A host that reports any of these as anything other + * than `false` cannot back the lane, because a case could then deliver physical input without anyone noticing. + */ +export const accessibleLaneMaskedFeatures = ['keyboard', 'physicalClick', 'wheel'] as const; +export type AccessibleLaneMaskedFeature = (typeof accessibleLaneMaskedFeatures)[number]; + +/** The masked features a host still offers. A missing report counts as unmasked, so the check fails closed. */ +export function unmaskedAccessibleFeatures(features: unknown): readonly AccessibleLaneMaskedFeature[] { + const reported = typeof features === 'object' && features !== null ? (features as Readonly>) : {}; + return accessibleLaneMaskedFeatures.filter((feature) => reported[feature] !== false); +} + export type DesktopPlatforms = | readonly [DesktopEndpoint, ...DesktopEndpoint[]] | readonly [`!${DesktopEndpoint}`, ...`!${DesktopEndpoint}`[]]; @@ -112,6 +127,78 @@ export function validateDesktopCaseOptions(value: unknown, source = 'desktop tes }; } +/** + * Validates the options of a group. A group scopes the cases inside it, so it can narrow where they run and add what they + * require, but a gap describes one case and cannot be declared on a group. + */ +export function validateDesktopGroupOptions(value: unknown, source = 'desktop group options'): ResolvedDesktopCaseOptions { + const options = validateDesktopCaseOptions(value, source); + if (options.unsupported || options.quarantine) { + throw new TypeError(`${source} cannot declare unsupported or quarantine; declare the gap on the case.`); + } + return options; +} + +/** + * Composes the options of an enclosing group with those of a nested group or case. Requirements add up, platforms and + * scope values intersect, and `input` can only narrow, so nothing inside a group can claim less than the group's own setup + * needs. A combination that could never run is an error rather than a silent exclusion. + */ +export function mergeDesktopCaseOptions( + parent: ResolvedDesktopCaseOptions | undefined, + child: ResolvedDesktopCaseOptions, + source = 'desktop test options', +): ResolvedDesktopCaseOptions { + if (!parent) return child; + const platforms = endpoints.filter( + (endpoint) => matchesDesktopPlatforms(parent.platforms, endpoint) && matchesDesktopPlatforms(child.platforms, endpoint), + ); + if (platforms.length === 0) throw new TypeError(`${source}.platforms do not overlap the enclosing group's, so it could never run.`); + const scope = intersectScope(parent.scope, child.scope, source); + const input = mergeDeclaredInput(parent, child, source); + return validateDesktopCaseOptions( + { + ...(input ? { input } : {}), + requires: [...new Set([...(parent.requires ?? []), ...(child.requires ?? [])])], + ...(platforms.length < endpoints.length ? { platforms } : {}), + ...(scope ? { scope } : {}), + ...(child.unsupported ? { unsupported: child.unsupported } : {}), + ...(child.quarantine ? { quarantine: child.quarantine } : {}), + }, + source, + ); +} + +function mergeDeclaredInput( + parent: ResolvedDesktopCaseOptions, + child: ResolvedDesktopCaseOptions, + source: string, +): DesktopInput | undefined { + // An explicit `both` narrows nothing, so it must not stop a nested case from deriving a transport. + const enclosing = parent.inputSource === 'declared' && parent.input !== 'both' ? parent.input : undefined; + if (child.inputSource !== 'declared') return enclosing; + if (enclosing && child.input !== enclosing) + throw new TypeError(`${source}.input "${child.input}" cannot change the enclosing group's "${enclosing}".`); + return child.input; +} + +function intersectScope( + parent: DesktopCaseOptions['scope'], + child: DesktopCaseOptions['scope'], + source: string, +): Record | undefined { + if (!parent && !child) return undefined; + const merged: Record = {}; + for (const key of new Set([...Object.keys(parent ?? {}), ...Object.keys(child ?? {})])) { + const left = parent?.[key]; + const right = child?.[key]; + const values = left && right ? left.filter((value) => right.includes(value)) : [...(left ?? right ?? [])]; + if (values.length === 0) throw new TypeError(`${source}.scope.${key} does not overlap the enclosing group's, so it could never run.`); + merged[key] = values; + } + return merged; +} + export function matchesDesktopPlatforms(platforms: DesktopPlatforms | undefined, endpoint: DesktopEndpoint): boolean { if (platforms === undefined) return true; const negative = platforms[0].startsWith('!'); @@ -119,6 +206,67 @@ export function matchesDesktopPlatforms(platforms: DesktopPlatforms | undefined, return negative ? !values.includes(endpoint) : values.includes(endpoint); } +/** Allowed values per scope key. */ +export type DesktopScopeDomains = Readonly>; +/** The one value a target holds for each scope key it supplies. */ +export type DesktopScopeValues = Readonly>; + +const builtInScopeDomains: DesktopScopeDomains = Object.freeze({ 'rn.renderer': Object.freeze(['fabric', 'paper']) }); + +/** + * The built-in domains plus any custom ones. A consumer adds its own keys, such as a product flavor, but `rn.*` belongs to + * the framework so a test written for `rn.renderer` means the same thing everywhere. + */ +export function resolveDesktopScopeDomains(custom?: unknown, source = 'testScopes'): DesktopScopeDomains { + if (custom === undefined) return builtInScopeDomains; + const resolved: Record = { ...builtInScopeDomains }; + for (const [key, values] of Object.entries(object(custom, source))) { + if (key.startsWith('rn.')) throw new TypeError(`${source} cannot declare "${key}"; the rn.* scope keys are reserved.`); + resolved[key] = Object.freeze([...scopeEntry(key, values, source)]); + } + return Object.freeze(resolved); +} + +/** Checks that every key a target supplies is a known domain and that its value belongs to it. */ +export function resolveDesktopScopeValues(supplied: unknown, domains: DesktopScopeDomains, source = 'testScope'): DesktopScopeValues { + const resolved: Record = {}; + for (const [key, value] of Object.entries(object(supplied, source))) { + const domain = ownDomain(domains, key); + if (!domain) throw new TypeError(`${source} uses unknown scope key "${key}".`); + if (typeof value !== 'string' || !domain.includes(value)) throw new TypeError(`${source}.${key} must be one of: ${domain.join(', ')}.`); + resolved[key] = value; + } + return Object.freeze(resolved); +} + +/** + * Values OR within a key and keys AND. An undeclared key or value, or a key the target does not supply, is a + * configuration error rather than an exclusion, so a typo cannot silently drop a case. Every key is checked before the + * result is returned. + */ +export function matchesDesktopScope( + scope: DesktopCaseOptions['scope'], + supplied: DesktopScopeValues, + domains: DesktopScopeDomains, + source = 'desktop test scope', +): boolean { + let matches = true; + for (const [key, values] of Object.entries(scope ?? {})) { + const domain = ownDomain(domains, key); + if (!domain) throw new TypeError(`${source} uses unknown scope key "${key}".`); + const unknown = values.find((value) => !domain.includes(value)); + if (unknown !== undefined) throw new TypeError(`${source}.${key} uses "${unknown}", which is not one of: ${domain.join(', ')}.`); + const actual = Object.hasOwn(supplied, key) ? supplied[key] : undefined; + if (actual === undefined) throw new TypeError(`${source}.${key} is not supplied by this target.`); + if (!values.includes(actual)) matches = false; + } + return matches; +} + +function ownDomain(domains: DesktopScopeDomains, key: string): readonly string[] | undefined { + return Object.hasOwn(domains, key) ? domains[key] : undefined; +} + function validatePlatforms(value: unknown, source: string, required = false): void { if (value === undefined && !required) return; if (!Array.isArray(value) || value.length === 0) throw new TypeError(`${source} must be a non-empty platform list.`); @@ -138,19 +286,21 @@ function validatePlatforms(value: unknown, source: string, required = false): vo function validateScope(value: unknown, source: string): void { if (value === undefined) return; - const scope = object(value, source); - for (const [key, values] of Object.entries(scope)) { - if (!/^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)*$/.test(key)) throw new TypeError(`${source} contains invalid key "${key}".`); - if (!Array.isArray(values) || values.length === 0) throw new TypeError(`${source}.${key} must be a non-empty value list.`); - const seen = new Set(); - for (const item of values) { - if (typeof item !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(item)) { - throw new TypeError(`${source}.${key} contains an invalid value.`); - } - if (seen.has(item)) throw new TypeError(`${source}.${key} contains duplicate "${item}".`); - seen.add(item); + for (const [key, values] of Object.entries(object(value, source))) scopeEntry(key, values, source); +} + +function scopeEntry(key: string, values: unknown, source: string): readonly string[] { + if (!/^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)*$/.test(key)) throw new TypeError(`${source} contains invalid key "${key}".`); + if (!Array.isArray(values) || values.length === 0) throw new TypeError(`${source}.${key} must be a non-empty value list.`); + const seen = new Set(); + for (const item of values) { + if (typeof item !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(item)) { + throw new TypeError(`${source}.${key} contains an invalid value.`); } + if (seen.has(item)) throw new TypeError(`${source}.${key} contains duplicate "${item}".`); + seen.add(item); } + return values as readonly string[]; } function object(value: unknown, source: string): Record { diff --git a/packages/agentic/desktop-driver/src/authoring/selection.test.ts b/packages/agentic/desktop-driver/src/authoring/selection.test.ts new file mode 100644 index 0000000000..2a8505dafc --- /dev/null +++ b/packages/agentic/desktop-driver/src/authoring/selection.test.ts @@ -0,0 +1,96 @@ +import { resolveDesktopScopeDomains, resolveDesktopScopeValues, validateDesktopCaseOptions } from './input.js'; +import { decideDesktopCase, missingDesktopRequirements, resolveDesktopClickMode } from './selection.js'; +import type { DesktopCaseTarget } from './selection.js'; + +describe('decideDesktopCase', () => { + const domains = resolveDesktopScopeDomains(); + const target = (overrides: Partial = {}): DesktopCaseTarget => ({ + endpoint: 'windows', + domains, + values: resolveDesktopScopeValues({ 'rn.renderer': 'fabric' }, domains), + now: Date.parse('2026-01-01T00:00:00Z'), + ...overrides, + }); + const decide = (options: object, overrides?: Partial) => + decideDesktopCase(validateDesktopCaseOptions(options), target(overrides), 'story / case'); + + test('runs a case that applies, and ignores the input outside a lane', () => { + expect(decide({})).toEqual({ kind: 'run' }); + expect(decide({ requires: ['keyboard'] })).toEqual({ kind: 'run' }); + expect(decide({ requires: ['keyboard'] }, { lane: 'physical' })).toEqual({ kind: 'run' }); + }); + + test('excludes by the first applicable reason: platform, then scope, then input', () => { + const options = { platforms: ['macos'], scope: { 'rn.renderer': ['paper'] }, requires: ['keyboard'] }; + expect(decide(options, { lane: 'accessible' })).toEqual({ kind: 'excluded', reason: 'platform' }); + expect(decide({ ...options, platforms: ['windows'] }, { lane: 'accessible' })).toEqual({ kind: 'excluded', reason: 'scope' }); + expect(decide({ ...options, platforms: ['windows'], scope: { 'rn.renderer': ['fabric'] } }, { lane: 'accessible' })).toEqual({ + kind: 'excluded', + reason: 'input', + }); + }); + + test('keeps a case that runs in both lanes', () => { + expect(decide({ requires: ['click'] }, { lane: 'accessible' })).toEqual({ kind: 'run' }); + expect(decide({ requires: ['click'] }, { lane: 'physical' })).toEqual({ kind: 'run' }); + }); + + test('resolves a declared unsupported case as a gap instead of running or excluding it', () => { + const unsupported = { platforms: ['windows'], reason: 'Not projected yet.' }; + expect(decide({ unsupported })).toEqual({ kind: 'gap', gap: { kind: 'unsupported', reason: 'Not projected yet.' } }); + expect(decide({ unsupported }, { endpoint: 'macos' })).toEqual({ kind: 'run' }); + }); + + test('applies a quarantine to its lane and platforms, and rejects an expired one before anything runs', () => { + const quarantine = { issue: '#9', reason: 'Flaky.', expires: '2026-06-01', input: 'physical', platforms: ['windows'] }; + expect(decide({ quarantine }, { lane: 'physical' })).toEqual({ kind: 'gap', gap: { kind: 'quarantine', reason: '#9: Flaky.' } }); + expect(decide({ quarantine }, { lane: 'accessible' })).toEqual({ kind: 'run' }); + expect(decide({ quarantine })).toEqual({ kind: 'run' }); + expect(decide({ quarantine }, { endpoint: 'win32', lane: 'physical' })).toEqual({ kind: 'run' }); + expect(() => decide({ quarantine }, { lane: 'physical', now: Date.parse('2026-06-02T00:00:00Z') })).toThrow( + 'Quarantine #9 expired on 2026-06-01.', + ); + }); +}); + +describe('click mode and requirement preflight', () => { + test('adapts a click requirement to the lane without falling back to the other transport', () => { + const descriptor = validateDesktopCaseOptions({ requires: ['click'] }); + const accessibleMode = resolveDesktopClickMode(descriptor, 'auto', 'accessible'); + const physicalMode = resolveDesktopClickMode(descriptor, 'auto', 'physical'); + expect(accessibleMode).toBe('accessibility'); + expect(physicalMode).toBe('physical'); + expect(missingDesktopRequirements(descriptor, { accessibilityClick: true, physicalClick: false }, physicalMode)).toEqual(['click']); + expect(missingDesktopRequirements(descriptor, { accessibilityClick: false, physicalClick: true }, accessibleMode)).toEqual(['click']); + expect(missingDesktopRequirements(descriptor, { accessibilityClick: true }, accessibleMode)).toEqual([]); + }); + + test('outside a lane only an accessible case pins its click transport, and the rest keep the configured mode', () => { + expect(resolveDesktopClickMode(validateDesktopCaseOptions({ requires: ['accessibility-click'] }), 'auto', undefined)).toBe( + 'accessibility', + ); + expect(resolveDesktopClickMode(validateDesktopCaseOptions({ input: 'accessible' }), 'physical', undefined)).toBe('accessibility'); + // A physical case must not pin physical click, or a host that masks it fails the session instead of reporting unsupported. + expect(resolveDesktopClickMode(validateDesktopCaseOptions({ requires: ['physical-click'] }), 'auto', undefined)).toBe('auto'); + expect(resolveDesktopClickMode(validateDesktopCaseOptions({ requires: ['keyboard'] }), 'auto', undefined)).toBe('auto'); + expect(resolveDesktopClickMode(validateDesktopCaseOptions({ requires: ['click'] }), 'physical', undefined)).toBe('physical'); + expect(resolveDesktopClickMode(undefined, 'auto', undefined)).toBe('auto'); + }); + + test('an automatic click needs either transport, and an absent report meets nothing', () => { + const click = validateDesktopCaseOptions({ requires: ['click'] }); + expect(missingDesktopRequirements(click, { physicalClick: false, accessibilityClick: false }, 'auto')).toEqual(['click']); + expect(missingDesktopRequirements(click, { physicalClick: true }, 'auto')).toEqual([]); + expect(missingDesktopRequirements(validateDesktopCaseOptions({ requires: ['keyboard', 'focus'] }), undefined, 'auto')).toEqual([ + 'keyboard', + 'focus', + ]); + expect( + missingDesktopRequirements( + validateDesktopCaseOptions({ requires: ['element-screenshot', 'set-window-rect'] }), + { elementScreenshot: true, setWindowRect: false }, + 'auto', + ), + ).toEqual(['set-window-rect']); + }); +}); diff --git a/packages/agentic/desktop-driver/src/authoring/selection.ts b/packages/agentic/desktop-driver/src/authoring/selection.ts new file mode 100644 index 0000000000..383ce5896f --- /dev/null +++ b/packages/agentic/desktop-driver/src/authoring/selection.ts @@ -0,0 +1,116 @@ +import type { DesktopEndpoint } from '../protocol/types.js'; +import { matchesDesktopPlatforms, matchesDesktopScope } from './input.js'; +import type { DesktopInputLane, DesktopRequirement, DesktopScopeDomains, DesktopScopeValues, ResolvedDesktopCaseOptions } from './input.js'; + +/** Why a declared case is not part of a run. Excluded cases are not tests, so they are counted and never reported as skipped. */ +export type DesktopExclusionReason = 'platform' | 'scope' | 'input'; + +/** A gap the author declared in advance. The case is selected and reported as unsupported, not executed. */ +export type DesktopDeclaredGap = { kind: 'unsupported' | 'quarantine'; reason: string }; + +/** What a case is judged against: the endpoint and scope values of the target, and the input lane of the run if it has one. */ +export type DesktopCaseTarget = { + endpoint: DesktopEndpoint; + lane?: DesktopInputLane; + domains: DesktopScopeDomains; + values: DesktopScopeValues; + /** Clock for quarantine expiry. */ + now?: number; +}; + +export type DesktopCaseDecision = + | { kind: 'excluded'; reason: DesktopExclusionReason } + | { kind: 'gap'; gap: DesktopDeclaredGap } + | { kind: 'run' }; + +/** + * Decides one declared case before it executes. The first applicable reason wins, in platform, scope, input order, so a + * case never counts against two exclusions. A declared gap is resolved here too, which is why an expired quarantine + * stops the run before anything executes. + */ +export function decideDesktopCase(descriptor: ResolvedDesktopCaseOptions, target: DesktopCaseTarget, source: string): DesktopCaseDecision { + if (!matchesDesktopPlatforms(descriptor.platforms, target.endpoint)) return { kind: 'excluded', reason: 'platform' }; + if (!matchesDesktopScope(descriptor.scope, target.values, target.domains, `${source} scope`)) + return { kind: 'excluded', reason: 'scope' }; + if (target.lane && descriptor.input !== 'both' && descriptor.input !== target.lane) return { kind: 'excluded', reason: 'input' }; + const gap = declaredGap(descriptor, target); + return gap ? { kind: 'gap', gap } : { kind: 'run' }; +} + +/** How a session turns a click into input. `auto` lets the host pick; the other two pin one transport. */ +export type DesktopCaseClickMode = 'auto' | 'physical' | 'accessibility'; + +/** + * The click mode a case runs under. A lane decides it outright, so a `click` requirement is an accessible click in the + * accessible lane and a physical click in the physical lane. Without a lane only an accessible case pins its transport, + * so it never moves the pointer. Any other case keeps the configured mode: a physical case on a host that masks physical + * click is then reported unsupported by the requirement preflight, as the skip it replaces was, instead of failing to + * create its session. + */ +export function resolveDesktopClickMode( + descriptor: ResolvedDesktopCaseOptions | undefined, + configured: DesktopCaseClickMode, + lane: DesktopInputLane | undefined, +): DesktopCaseClickMode { + if (lane === 'accessible') return 'accessibility'; + if (lane === 'physical') return 'physical'; + if (descriptor?.input === 'accessible' || descriptor?.requires?.includes('accessibility-click')) return 'accessibility'; + return configured; +} + +/** The requirements a host cannot meet. `features` is what the host reported for the session; an absent report meets nothing. */ +export function missingDesktopRequirements( + descriptor: ResolvedDesktopCaseOptions | undefined, + features: Readonly> | undefined, + clickMode: DesktopCaseClickMode, +): DesktopRequirement[] { + if (!descriptor?.requires?.length) return []; + const missing: DesktopRequirement[] = []; + for (const requirement of descriptor.requires) { + if (requirement === 'click') { + const met = + clickMode === 'physical' + ? features?.physicalClick === true + : clickMode === 'accessibility' + ? features?.accessibilityClick === true + : features?.physicalClick === true || features?.accessibilityClick === true; + if (!met) missing.push(requirement); + } else if (features?.[requirementFeature[requirement]] !== true) { + missing.push(requirement); + } + } + return missing; +} + +/** The host feature that backs each requirement other than `click`, which depends on the click mode. */ +const requirementFeature: Record, string> = { + 'accessibility-actions': 'accessibilityActions', + 'accessibility-click': 'accessibilityClick', + 'element-screenshot': 'elementScreenshot', + focus: 'focus', + keyboard: 'keyboard', + 'physical-click': 'physicalClick', + screenshot: 'screenshot', + 'set-window-rect': 'setWindowRect', + wheel: 'wheel', +}; + +function declaredGap( + descriptor: ResolvedDesktopCaseOptions, + { endpoint, lane, now = Date.now() }: DesktopCaseTarget, +): DesktopDeclaredGap | undefined { + if (descriptor.unsupported && matchesDesktopPlatforms(descriptor.unsupported.platforms, endpoint)) { + return { kind: 'unsupported', reason: descriptor.unsupported.reason }; + } + const quarantine = descriptor.quarantine; + if ( + quarantine && + (!quarantine.platforms || matchesDesktopPlatforms(quarantine.platforms, endpoint)) && + (!quarantine.input || quarantine.input === lane) + ) { + if (Date.parse(`${quarantine.expires}T00:00:00Z`) < now) + throw new Error(`Quarantine ${quarantine.issue} expired on ${quarantine.expires}.`); + return { kind: 'quarantine', reason: `${quarantine.issue}: ${quarantine.reason}` }; + } + return undefined; +} diff --git a/packages/agentic/desktop-driver/src/index.ts b/packages/agentic/desktop-driver/src/index.ts index 3e556374bc..0479787d78 100644 --- a/packages/agentic/desktop-driver/src/index.ts +++ b/packages/agentic/desktop-driver/src/index.ts @@ -32,15 +32,39 @@ export { desktopStoryPlatforms, validateDesktopStoryTests, } from './authoring/storyTests.js'; -export { desktopRequirements, matchesDesktopPlatforms, resolveDesktopInput, validateDesktopCaseOptions } from './authoring/input.js'; +export { + accessibleLaneMaskedFeatures, + desktopInputLanes, + desktopRequirements, + matchesDesktopPlatforms, + matchesDesktopScope, + mergeDesktopCaseOptions, + resolveDesktopInput, + resolveDesktopScopeDomains, + resolveDesktopScopeValues, + unmaskedAccessibleFeatures, + validateDesktopCaseOptions, + validateDesktopGroupOptions, +} from './authoring/input.js'; export type { + AccessibleLaneMaskedFeature, DesktopCaseOptions, DesktopInput, DesktopInputLane, DesktopPlatforms, DesktopRequirement, + DesktopScopeDomains, + DesktopScopeValues, ResolvedDesktopCaseOptions, } from './authoring/input.js'; +export { decideDesktopCase, missingDesktopRequirements, resolveDesktopClickMode } from './authoring/selection.js'; +export type { + DesktopCaseClickMode, + DesktopCaseDecision, + DesktopCaseTarget, + DesktopDeclaredGap, + DesktopExclusionReason, +} from './authoring/selection.js'; export type { DesktopStoryCapability, DesktopStoryExpectation, diff --git a/packages/agentic/desktop-test/package.json b/packages/agentic/desktop-test/package.json index f86c5cecd6..8a01e622bb 100644 --- a/packages/agentic/desktop-test/package.json +++ b/packages/agentic/desktop-test/package.json @@ -55,6 +55,11 @@ "import": "./lib/driverService.js", "default": "./src/driverService.ts" }, + "./jasmine": { + "types": "./lib/jasmine/index.d.ts", + "import": "./lib/jasmine/index.js", + "default": "./src/jasmine/index.ts" + }, "./wdio": { "types": "./lib/wdio/index.d.ts", "import": "./lib/wdio/index.js", diff --git a/packages/agentic/desktop-test/references/commands.md b/packages/agentic/desktop-test/references/commands.md index 47666999c4..817553b4bc 100644 --- a/packages/agentic/desktop-test/references/commands.md +++ b/packages/agentic/desktop-test/references/commands.md @@ -10,14 +10,15 @@ owner_paths: Expose `desktop-test` from the consumer workspace and run through that workspace's declared script. -| Command | Purpose | -| -------------------------------- | ---------------------------------------------------- | -| `build-driver` | Explicitly build or verify the native helper | -| `prep`, `bundle`, `build`, `run` | Run the selected target adapter stage | -| `test --suite --list` | Resolve groups without launching a target | -| `test --suite ` | Run the selected WDIO suite | -| `serve --reporter json` | Start the real driver listener and required services | -| `serve --agent` | Also require grants and publish an agent descriptor | +| Command | Purpose | +| ------------------------------------ | ---------------------------------------------------- | +| `build-driver` | Explicitly build or verify the native helper | +| `prep`, `bundle`, `build`, `run` | Run the selected target adapter stage | +| `test --suite --list` | Resolve groups without launching a target | +| `test --suite ` | Run the selected WDIO suite | +| `test --suite --input ` | Run one input lane (`accessible` or `physical`) | +| `serve --reporter json` | Start the real driver listener and required services | +| `serve --agent` | Also require grants and publish an agent descriptor | Global reporter options are `--reporter auto|plain|github-actions|json|ndjson` and `--json` (an alias for `--reporter json`). Mode precedence is CLI flag, @@ -58,6 +59,24 @@ For macOS RN Test App projects, run bundle before `prep`; CocoaPods embeds only resources present during pod installation. Changing the renderer requires rerunning `prep`. See [platform setup](platforms.md). +## Input lanes + +`test --input accessible|physical` runs one input lane. `DESKTOP_INPUT` supplies +the lane when the flag is absent, and an empty or unknown value is an error. The +`accessible` lane starts the native host with keyboard, physical click, and wheel +disabled and runs the tests that need no physical input; the `physical` lane +runs tests that need it plus those that need neither. Tests declare which with +the [`./jasmine` helpers](wdio-suites.md#declare-input-and-scope-in-jasmine-specs). +A lane requires a Jasmine WDIO suite (or the Storybook suite, whose integration +applies the lane itself) and rejects `--list`, which resolves spec files without +regard to a lane. Without a lane every declared test runs, filtered by `platforms` +and `scope`. See [input lanes in reporting](reporting.md#input-lanes) for how a +lane run is judged. + +An imported integration that serves its own driver receives the lane and owns +masking that driver. A host supplied through `hostFactory` is not masked by the +harness, so the suite verifies the mask before running anything. + The Storybook `csf-wdio` suite does not support generic `--list`. Discover story IDs with the Storybook integration's manifest/CLI, then select authored tests with its test filter. Use `storybook-desktop` smoke commands for a complete diff --git a/packages/agentic/desktop-test/references/configuration.md b/packages/agentic/desktop-test/references/configuration.md index 7895d23d4c..4a780442a7 100644 --- a/packages/agentic/desktop-test/references/configuration.md +++ b/packages/agentic/desktop-test/references/configuration.md @@ -37,6 +37,34 @@ The config schema rejects unknown fields, cycles, and unmatched spec patterns. Spec paths resolve against the suite project's root; target references and hooks resolve against their declaring config. +## Scope domains + +A test can declare `scope` values so it runs only on some targets, such as one +renderer or product flavor. Domains are declared in config. `rn.renderer` +(`fabric`, `paper`) is built in and comes from the target's React Native host; +the `rn.*` keys are reserved for the framework. A consumer that is not React +Native, or that needs another dimension, lists its own keys in `testScopes`, each +with a non-empty list of allowed values, and each target sets its own value in +`testScope`: + +```ts +export default defineDesktopConfig({ + schemaVersion: 1, + projectRoot: new URL('.', import.meta.url), + testScopes: { flavor: ['stable', 'preview'] }, + targets: { + windows: { + host: { kind: 'installed', application: { executablePath: 'C:\\apps\\Sample.exe', windowTitle: 'Sample' } }, + testScope: { flavor: 'preview' }, + }, + }, +}); +``` + +A `testScope` key that is not declared, a value outside its domain, or an attempt +to set an `rn.*` key is a configuration error. So is a test scoped on a key its +target does not supply: a typo or a missing value must not silently drop tests. + ## Helper policy `nativeDriver` accepts only `cacheRoot`, `helperPath`, and `installRoot`. diff --git a/packages/agentic/desktop-test/references/contributing.md b/packages/agentic/desktop-test/references/contributing.md index d5f7175b95..ee9a698cc6 100644 --- a/packages/agentic/desktop-test/references/contributing.md +++ b/packages/agentic/desktop-test/references/contributing.md @@ -12,7 +12,18 @@ CLI stages are composed in `src/cli.ts`; process ownership is in `src/supervisor.ts`; shared native service, recovery, authority, descriptor and settled shutdown composition lives in the real `src/driverService.ts` module (also used by the Storybook channel child); reports are implemented in `src/reporting/`; and WDIO -session wiring is in `src/wdio/`. +session wiring is in `src/wdio/`. The `./jasmine` entry (`src/jasmine/`) holds the +declaration helpers and the selection adapter that WDIO Jasmine suites use; input +lane accounting is in `src/reporting/lane.ts`. Descriptor validation and per-case +decisions come from `@fluentui-react-native/desktop-driver/authoring`, not from +copies here. + +`src/config.ts` is also loaded from source through the CommonJS prebuild fallback +that Storybook uses for `desktop.config.mts`, which cannot resolve desktop-driver's +`.js` specifiers. Its runtime imports are therefore Node built-ins, sibling `.ts` +modules and the leaf `@fluentui-react-native/desktop-driver/authoring/input`; import +everything else from desktop-driver as types only. The storybook-desktop +`sourceConfig` test guards this. Add contract coverage for schema, lifecycle, reporting protocol, and failure paths. Do not move Storybook-specific story extraction into this package. diff --git a/packages/agentic/desktop-test/references/reporting.md b/packages/agentic/desktop-test/references/reporting.md index 164b05dc64..4e91dc7319 100644 --- a/packages/agentic/desktop-test/references/reporting.md +++ b/packages/agentic/desktop-test/references/reporting.md @@ -65,6 +65,58 @@ directory or render a duplicate summary. Story-specific readiness, authenticated channel events, and callback execution are owned by `@fluentui-react-native/storybook-desktop`. +## Input lanes + +A `test` run can execute one input lane (see [commands](commands.md#input-lanes)). +Its report records the lane, counts what the run left out before executing, and +holds every selected test to its declared outcome. `run.json` carries the lane as +`{ input, source, masking?, policyEnforced }`: + +- `input`: `accessible` or `physical`. +- `source`: `option` for `--input`, `environment` for `DESKTOP_INPUT`, or `driver` + when the lane was read from the driver instance a `test` attached to. +- `masking` (accessible only): `host-masked` when this command started the host + with keyboard, physical click, and wheel disabled, or `verified-masked` when it + attached to a host and verified that mask before running anything. +- `policyEnforced`: always `false`. The report records what was requested and how + the host was masked; it never claims what input a test delivered, so a physical + lane is labelled a pilot. + +Tests that a lane, a `platforms` list, or a `scope` leaves out are not tests, so +they are never reported as skipped. A producer announces them in `selection` +events, and `run.json` sums them as `selection.excluded: { platform, scope, input }`. +A case counts once, under the first reason that applies in that order, and only if +it matched the user's own filters, such as a story filter. A `selection` event +with a `key` is counted once, so a retried worker that announces the same +exclusions again does not double them. A run whose producers reported none has no +`selection`. + +A lane run promised to execute its whole selection, so its verdict is stricter: + +- `failed`: a failed test or an infrastructure failure, as in any run. +- `incomplete`: a selected test was skipped at run time, was `unsupported` with no + declaration (the host did not meet a `requires`), or no test executed. `optional` + never excuses these, so a lane that runs nothing is not reported `skipped`. +- `passed`: otherwise. A test declared `unsupported` or `quarantine` is reported + unsupported without making the run incomplete. Summaries count them as + `N unsupported (N declared, N quarantined)` so an undeclared gap stands out. + +`failed` outranks `incomplete`, which outranks `passed`. A run without a lane keeps +its existing verdict. + +The plain and GitHub summaries show the lane and exclusions on one line, such as +`Selection: input accessible (host-masked) from --input; excluded 4 by platform, 0 by scope, 12 by input`. +The JUnit file records the lane as the properties `desktop.input`, +`desktop.input.label`, and `desktop.input.source`, the exclusions as +`desktop.excluded.platform`, `desktop.excluded.scope`, and `desktop.excluded.input`, +and `desktop.declaredUnsupported` and `desktop.quarantined` when present. The label +is `accessible (host-masked)`, `accessible (verified-masked)`, or `physical pilot`. + +A delegated command, which inherits its parent's collector, sends its exclusions to +the parent but not its lane, so the parent's `run.json` has no `lane`. The command +still holds its own cohort to the lane: a Storybook `test` reports an incomplete +cohort to the parent when a selected test has no declared gap and did not run. + ## Persisted read queries The direct `@fluentui-react-native/desktop-test/reporting/run-queries` entry @@ -117,7 +169,8 @@ external integration); `addSecret` can update the read redactor. The collector redacts before persistence, and queries redact again before clipping text. Readers submit only an opaque run ID, a declared operation, and its bounded -parameters. `status` returns complete counts and terminal/coverage state. +parameters. `status` returns complete counts, terminal/coverage state, and the +run's input `lane` and `excluded` counts when it has them. `tests`, `work`, `failures`, and `artifacts` return pages with a default limit of 10 and maximum of 25, capped at 16 KiB. Page IDs do not expose producer IDs or paths. Text fields are capped at 1 KiB after redaction. `omissions.items` counts diff --git a/packages/agentic/desktop-test/references/wdio-suites.md b/packages/agentic/desktop-test/references/wdio-suites.md index bf6a716e74..fa80879ec3 100644 --- a/packages/agentic/desktop-test/references/wdio-suites.md +++ b/packages/agentic/desktop-test/references/wdio-suites.md @@ -40,3 +40,68 @@ Prefer unique ids. A suite that depends on first-match lookup can set Component story callback extraction is owned by `@fluentui-react-native/storybook-desktop`; use this suite guide for ordinary WDIO suites and app-level E2E adapters. + +## Declare input and scope in Jasmine specs + +Declare each test with `desktopIt` and group tests with `desktopDescribe` from +`@fluentui-react-native/desktop-test/jasmine`. They take the same descriptor as +Storybook's `desktopIt` (`requires`, `input`, `platforms`, `scope`, `unsupported`, +`quarantine`), and the options argument is optional: + +```ts +import { desktopDescribe, desktopIt } from '@fluentui-react-native/desktop-test/jasmine'; + +desktopDescribe('Button', () => { + desktopIt('renders', async () => { + await expect(await $('~button')).toBeDisplayed(); + }); + desktopIt('activates with Enter', { requires: ['keyboard'] }, async () => { + await browser.keys('\uE007'); + }); + desktopDescribe('Windows behavior', { platforms: ['windows', 'win32'] }, () => { + desktopIt('takes focus', { requires: ['focus'] }, async () => { + await expect(await $('~button')).toBeFocused(); + }); + }); +}); +``` + +A group's options apply to everything inside it. Requirements add up, `platforms` +and `scope` intersect, and a nested declaration can only narrow its group, so a +combination that could never run is an error rather than a silent exclusion. A +group cannot declare `unsupported` or `quarantine`; declare the gap on the test. +A test's `input` comes from its `requires` unless declared; see "Declare how a case runs" in +`@fluentui-react-native/storybook-desktop/references/story-tests.md`. +A test that needs no input stays `both`. + +Run through `desktop-test test --suite `, the harness hands each WDIO worker +its endpoint, scope values, and optional lane as data in `DESKTOP_TEST_SELECTION`. +Once the session exists and before any test runs, the adapter filters Jasmine's +`specFilter`, so an excluded test never runs its hooks or body. Excluded tests +are counted in the run's selection totals, not reported as skipped. A declared +`unsupported` or active `quarantine` test is reported unsupported with its reason, +and an expired quarantine fails that test. A test whose `requires` the host cannot +meet is excluded the same way and reported unsupported. + +- **Lanes need declared tests.** An input lane runs only tests declared with + `desktopIt`. A test registered with plain `it` fails the run, which lists the first + ten, so migrate a whole file before running a lane against it. A plain `describe` + is fine; use `desktopDescribe` to scope its tests. Without a lane, plain specs run + unchanged. +- **Click mode is session-wide.** WDIO owns the session, so the lane sets + `furn:clickMode` once: accessibility click in the accessible lane, physical + click in the physical lane. A `click` requirement follows it. Without a lane the + session uses `auto`, which lets the host choose. +- **Untrustworthy selections fail closed.** A malformed selection, an unverifiable + Jasmine environment, or an accessible lane on a host that does not report keyboard, + physical click, and wheel disabled disables every test and fails the run. +- **A bad declaration never throws.** Jasmine only warns when a spec file fails to + load, so a throw would silently drop every test after it. An invalid declaration + registers a failing test that reports the problem. +- **Run outside the harness.** With no selection, such as plain `wdio`, every test + runs and descriptors are still validated. +- `desktopIt` takes no timeout argument; the suite's `timeoutMs` applies. + +The input-call lint of Storybook cases does not apply here. Page objects and helpers +hide the input a test sends, so the accessible lane's host mask is the safeguard that +fails a test that reaches for physical input. diff --git a/packages/agentic/desktop-test/src/cli.ts b/packages/agentic/desktop-test/src/cli.ts index 833b7b1c29..3258f8a45e 100644 --- a/packages/agentic/desktop-test/src/cli.ts +++ b/packages/agentic/desktop-test/src/cli.ts @@ -11,15 +11,17 @@ import { tolerateClosedDesktopStdout, } from '@fluentui-react-native/desktop-driver/cli/reporter'; import type { ResolvedDesktopReporterMode } from '@fluentui-react-native/desktop-driver/cli/reporter'; +import { desktopInputLanes } from '@fluentui-react-native/desktop-driver/authoring'; import { desktopStageCommands } from './adapters.js'; import { defaultAgentDescriptor } from './agentConnection.js'; import type { DesktopAgentConnectionOptions } from './agentConnection.js'; import { loadDesktopConfig, resolveDesktopSpecs, selectDesktopRenderer } from './config.js'; import type { DesktopStage } from './config.js'; +import { createDesktopInputLane, desktopInputEnvironment, requestedDesktopInputLane } from './reporting/lane.js'; import { createDesktopReportSession } from './reporting/session.js'; import { DesktopStopRequested } from './services.js'; import { DesktopSupervisor } from './supervisor.js'; -import { runDesktopWdioSuite } from './wdio/run.js'; +import { assertDesktopSuiteSupportsInput, runDesktopWdioSuite } from './wdio/run.js'; type AgentFlags = { agent?: boolean; descriptor?: string; root?: string }; @@ -53,7 +55,13 @@ export async function runDesktopTestCli(argv = process.argv): Promise { .command('test') .requiredOption('--suite ', 'configured suite') .option('--list', 'resolve exact spec groups without launching') - .action((options: { suite: string; list?: boolean }) => execute('test', options)); + .addOption( + new Option( + '--input ', + `run one input lane: accessible (keyboard, physical click and wheel masked) or physical; falls back to ${desktopInputEnvironment}`, + ).choices([...desktopInputLanes]), + ) + .action((options: { suite: string; list?: boolean; input?: string }) => execute('test', options)); let reporterMode: ResolvedDesktopReporterMode = 'plain'; let reporterResolutionError: unknown; try { @@ -82,7 +90,7 @@ export async function runDesktopTestCli(argv = process.argv): Promise { async function execute( action: DesktopStage | 'build-driver' | 'serve' | 'test', - test?: { suite: string; list?: boolean }, + test?: { suite: string; list?: boolean; input?: string }, agentFlags: AgentFlags = {}, ): Promise { const options = program.opts<{ @@ -118,6 +126,11 @@ export async function runDesktopTestCli(argv = process.argv): Promise { if (options.paper) target = selectDesktopRenderer(target, 'paper'); const suite = test ? config.config.suites?.[test.suite] : undefined; if (test && !suite) throw Object.assign(new Error(`Unknown suite "${test.suite}".`), { code: 'config' }); + if (test?.list && test.input !== undefined) { + throw Object.assign(new TypeError('--list resolves spec files, which do not depend on the input lane; omit --input.'), { + code: 'config', + }); + } if (suite?.kind === 'wdio') { const specs = resolveDesktopSpecs(suite, target.endpoint, config.projectRoot); if (test?.list) { @@ -131,6 +144,12 @@ export async function runDesktopTestCli(argv = process.argv): Promise { return; } } else if (test?.list) throw Object.assign(new Error('--list currently requires a standalone WDIO suite.'), { code: 'config' }); + // Only `test` has a lane. The environment stands in for the flag, but never for `--list`, which returned above. + const requestedLane = test + ? requestedDesktopInputLane({ flag: test.input, environment: process.env[desktopInputEnvironment] }) + : undefined; + const input = requestedLane?.input; + if (suite?.kind === 'wdio') assertDesktopSuiteSupportsInput(test!.suite, suite, input); if (!agentFlags.agent && (agentFlags.descriptor !== undefined || agentFlags.root !== undefined)) throw Object.assign(new TypeError('--descriptor and --root require --agent.'), { code: 'config' }); // An integration serves through its own lifecycle, so it must publish the agent connection itself. @@ -156,6 +175,7 @@ export async function runDesktopTestCli(argv = process.argv): Promise { kind: action === 'test' ? 'tests' : 'lifecycle', optional: suite?.kind === 'wdio' && suite.optional, context: { endpoint: target.endpoint, renderer: target.host.renderer }, + ...(requestedLane ? { lane: createDesktopInputLane(requestedLane.input, requestedLane.source) } : {}), reporterMode, eventLevel, suppressFinalOutput: action === 'serve' ? () => ready : false, @@ -169,6 +189,7 @@ export async function runDesktopTestCli(argv = process.argv): Promise { reportSession: session, integration: config.config.integration, ...(agent ? { agent } : {}), + ...(input ? { input } : {}), }); session.onStop?.(() => owner.services.requestStop()); const interrupt = () => owner.services.requestStop(); @@ -196,6 +217,7 @@ export async function runDesktopTestCli(argv = process.argv): Promise { supervisor: owner, action, suite: test?.suite, + ...(input ? { input } : {}), ...(agent ? { agent } : {}), ...(action === 'serve' ? { reporterMode, onReady: markReady } : {}), }), @@ -219,7 +241,7 @@ export async function runDesktopTestCli(argv = process.argv): Promise { ); markReady(); await owner.services.guard(new Promise(() => undefined)); - } else if (suite?.kind === 'wdio') await runDesktopWdioSuite(owner, config, test!.suite); + } else if (suite?.kind === 'wdio') await runDesktopWdioSuite(owner, config, test!.suite, input); else throw new Error('The imported integration does not implement this suite.'); } } catch (error) { diff --git a/packages/agentic/desktop-test/src/config.ts b/packages/agentic/desktop-test/src/config.ts index edf975b6bb..79fcd0ee85 100644 --- a/packages/agentic/desktop-test/src/config.ts +++ b/packages/agentic/desktop-test/src/config.ts @@ -4,6 +4,10 @@ import { fileURLToPath, pathToFileURL } from 'node:url'; import { createHash } from 'node:crypto'; import type { DesktopEndpoint, DesktopRenderer, NativeDesktopApplicationDescriptor } from '@fluentui-react-native/desktop-driver'; +// The leaf subpath, not `/authoring`: a config file is also loaded from source through the CommonJS prebuild fallback, +// which cannot resolve the `.js` specifiers inside the authoring index. +import { resolveDesktopScopeDomains, resolveDesktopScopeValues } from '@fluentui-react-native/desktop-driver/authoring/input'; +import type { DesktopScopeDomains, DesktopScopeValues } from '@fluentui-react-native/desktop-driver/authoring/input'; import type { DesktopReporterMode } from '@fluentui-react-native/desktop-driver/cli/reporter'; import type { DesktopIntegration } from './integration.ts'; @@ -35,6 +39,11 @@ export type DesktopTargetConfig = { services?: readonly { id: string; command: DesktopCommand; readyUrl?: string }[]; launchTimeoutMs?: number; nativeDriver?: { cacheRoot?: string; helperPath?: string; installRoot?: string }; + /** + * The value this target holds for each custom `testScopes` key a test may scope on, such as a product flavor. + * `rn.renderer` is not set here; it comes from `host.renderer`. + */ + testScope?: Readonly>; }; export type DesktopTargetReference = { use: { config: string; target: string } }; @@ -68,6 +77,11 @@ export type DesktopConfig = { suites?: Readonly>; integration?: DesktopIntegration; reporting?: DesktopReportingConfig; + /** + * Custom scope domains: each key lists the values a test may scope on, and every target sets its own in `testScope`. + * The `rn.*` keys belong to the framework, so a consumer that is not React Native simply declares its own. + */ + testScopes?: Readonly>; }; export type ResolvedDesktopTarget = DesktopTargetConfig & { @@ -84,6 +98,8 @@ export type ResolvedDesktopConfig = { projectRoot: string; targets: Readonly>; reporting: DesktopReportingConfig & { artifactsRoot: string }; + /** The built-in scope domains plus this config's own. */ + testScopes: DesktopScopeDomains; }; const configNames = ['desktop.config.mts', 'desktop.config.ts', 'desktop.config.mjs', 'desktop.config.js', 'desktop.config.cjs']; @@ -98,9 +114,10 @@ export function defineDesktopConfig(config: T): T { export function validateDesktopConfig(value: unknown): asserts value is DesktopConfig { const config = object(value, 'desktop config'); - keys(config, ['schemaVersion', 'projectRoot', 'targets', 'suites', 'integration', 'reporting'], 'desktop config'); + keys(config, ['schemaVersion', 'projectRoot', 'targets', 'suites', 'integration', 'reporting', 'testScopes'], 'desktop config'); if (config.schemaVersion !== 1) throw new TypeError('Desktop config schemaVersion must be 1.'); if (config.projectRoot !== undefined && !(config.projectRoot instanceof URL)) text(config.projectRoot, 'projectRoot'); + const scopeDomains = resolveDesktopScopeDomains(config.testScopes); const targets = object(config.targets, 'targets'); if (Object.keys(targets).length === 0) throw new TypeError('Desktop config must declare at least one target.'); for (const [id, entry] of Object.entries(targets)) { @@ -114,8 +131,9 @@ export function validateDesktopConfig(value: unknown): asserts value is DesktopC text(reference.target, `${id}.use.target`); continue; } - keys(target, ['endpoint', 'host', 'readiness', 'commands', 'services', 'launchTimeoutMs', 'nativeDriver'], `target ${id}`); + keys(target, ['endpoint', 'host', 'readiness', 'commands', 'services', 'launchTimeoutMs', 'nativeDriver', 'testScope'], `target ${id}`); oneOf(target.endpoint ?? id, endpoints, `${id}.endpoint`); + if (target.testScope !== undefined) validateTestScope(target.testScope, scopeDomains, id); const host = object(target.host, `${id}.host`); keys(host, ['kind', 'application', 'ownership', 'renderer', 'appConfigPath', 'component', 'windowTitle', 'bundleRoot'], `${id}.host`); oneOf(host.kind, ['installed', 'react-native-test-app', 'rex'], `${id}.host.kind`); @@ -241,6 +259,8 @@ export async function loadDesktopConfig(configPath?: string, cwd = process.cwd() } return pending; }; + const entry = await read(filename); + const testScopes = resolveDesktopScopeDomains(entry.config.testScopes); const resolve = async (filename: string, id: string, ancestry: readonly string[]): Promise => { const { config, root, file } = await read(filename); const key = `${file}#${id}`; @@ -248,6 +268,8 @@ export async function loadDesktopConfig(configPath?: string, cwd = process.cwd() const target = config.targets[id]; if (!target) throw new Error(`Unknown desktop target "${id}" in ${file}.`); if ('use' in target) return resolve(path.resolve(path.dirname(file), target.use.config), target.use.target, [...ancestry, key]); + // A referenced target was checked against its own config's domains; the config that uses it must declare them too. + if (target.testScope) resolveDesktopScopeValues(target.testScope, testScopes, `${id}.testScope`); const endpoint = target.endpoint ?? (id as DesktopEndpoint); const host = { ...target.host, @@ -276,17 +298,38 @@ export async function loadDesktopConfig(configPath?: string, cwd = process.cwd() : undefined, }); }; - const { config, root, file } = await read(filename); - const targets = Object.fromEntries(await Promise.all(Object.keys(config.targets).map(async (id) => [id, await resolve(file, id, [])]))); + const targets = Object.fromEntries( + await Promise.all(Object.keys(entry.config.targets).map(async (id) => [id, await resolve(entry.file, id, [])])), + ); return { - config, - configPath: file, - projectRoot: root, + config: entry.config, + configPath: entry.file, + projectRoot: entry.root, targets: Object.freeze(targets), - reporting: { ...config.reporting, artifactsRoot: path.resolve(root, config.reporting?.artifactsRoot ?? 'artifacts/desktop') }, + reporting: { + ...entry.config.reporting, + artifactsRoot: path.resolve(entry.root, entry.config.reporting?.artifactsRoot ?? 'artifacts/desktop'), + }, + testScopes, }; } +/** + * The values a target holds for the scope domains: its own `testScope` plus the renderer its host runs. A caller that + * knows the renderer better than the host configuration, such as a Storybook run switched to Paper, passes it. + */ +export function resolveDesktopTestScope( + domains: DesktopScopeDomains, + target?: Pick, + renderer: DesktopRenderer | undefined = target?.host.renderer, +): DesktopScopeValues { + return resolveDesktopScopeValues( + { ...target?.testScope, ...(renderer ? { 'rn.renderer': renderer } : {}) }, + domains, + target ? `${target.id}.testScope` : 'testScope', + ); +} + export function selectDesktopRenderer(target: ResolvedDesktopTarget, renderer: DesktopRenderer): ResolvedDesktopTarget { if (target.endpoint !== 'macos' || target.host.kind !== 'react-native-test-app') throw new Error('Renderer switching requires a macOS RN Test App target; regenerate with prep after switching.'); @@ -327,6 +370,14 @@ function resolvePaths(root: string, settings: object, fields: readonly string[]) ); } +function validateTestScope(value: unknown, domains: DesktopScopeDomains, id: string): void { + const scope = object(value, `${id}.testScope`); + for (const key of Object.keys(scope)) { + if (key.startsWith('rn.')) throw new TypeError(`${id}.testScope cannot set "${key}"; the rn.* values come from ${id}.host.`); + } + resolveDesktopScopeValues(scope, domains, `${id}.testScope`); +} + function validateApplication(value: unknown, context: string): void { const application = object(value, context); keys( diff --git a/packages/agentic/desktop-test/src/contracts.contract.cjs b/packages/agentic/desktop-test/src/contracts.contract.cjs index 7fef3c4e35..7182e5f8f1 100644 --- a/packages/agentic/desktop-test/src/contracts.contract.cjs +++ b/packages/agentic/desktop-test/src/contracts.contract.cjs @@ -49,7 +49,7 @@ async function main() { const { DesktopRunReporter } = await import(lib('reporting/collector.js')); const { NodeDesktopCommandRunner } = await import(lib('commands.js')); const { DesktopServiceScope, DesktopStopRequested } = await import(lib('services.js')); - const { loadDesktopConfig, defineDesktopConfig, resolveDesktopSpecs } = await import(lib('config.js')); + const { loadDesktopConfig, defineDesktopConfig, resolveDesktopSpecs, resolveDesktopTestScope } = await import(lib('config.js')); // Config resolution canonicalizes paths; macOS temporary directories are symlinked (/var -> /private/var). const root = fs.realpathSync.native(fs.mkdtempSync(path.join(os.tmpdir(), 'desktop-harness-contract-'))); const out = capture(); @@ -92,6 +92,43 @@ async function main() { 'export default {schemaVersion:1, targets:{win32:{use:{config:"cycle.mjs",target:"win32"}}}}', ); await assert.rejects(loadDesktopConfig(path.join(root, 'cycle.mjs')), /cycle/); + const flavored = (testScopes, testScope, host = app.targets.win32.host) => ({ + ...app, + ...(testScopes ? { testScopes } : {}), + targets: { win32: { host, ...(testScope ? { testScope } : {}) } }, + }); + const flavors = { 'product.flavor': ['alpha', 'beta'] }; + defineDesktopConfig(flavored(flavors, { 'product.flavor': 'beta' })); + assert.throws(() => defineDesktopConfig(flavored({ 'rn.renderer': ['x'] })), /reserved/); + assert.throws(() => defineDesktopConfig(flavored(flavors, { 'product.flavor': 'gamma' })), /must be one of: alpha, beta/); + assert.throws(() => defineDesktopConfig(flavored(flavors, { unknown: 'x' })), /unknown scope key "unknown"/); + assert.throws(() => defineDesktopConfig(flavored(flavors, { 'rn.renderer': 'paper' })), /cannot set "rn.renderer"/); + fs.mkdirSync(path.join(root, 'flavor')); + const paperHost = { ...app.targets.win32.host, renderer: 'paper' }; + fs.writeFileSync( + path.join(root, 'flavor', 'desktop.config.mjs'), + `export default ${JSON.stringify(flavored(flavors, { 'product.flavor': 'beta' }, paperHost))};`, + ); + const flavor = await loadDesktopConfig(path.join(root, 'flavor', 'desktop.config.mjs')); + assert.deepEqual({ ...flavor.testScopes }, { 'rn.renderer': ['fabric', 'paper'], ...flavors }); + assert.deepEqual( + { ...resolveDesktopTestScope(flavor.testScopes, flavor.targets.win32) }, + { 'product.flavor': 'beta', 'rn.renderer': 'paper' }, + ); + assert.deepEqual( + { ...resolveDesktopTestScope(flavor.testScopes, flavor.targets.win32, 'fabric') }, + { 'product.flavor': 'beta', 'rn.renderer': 'fabric' }, + ); + assert.deepEqual( + { ...resolveDesktopTestScope(direct.testScopes, direct.targets.win32) }, + {}, + 'no renderer is derived without a host renderer', + ); + fs.writeFileSync( + path.join(root, 'flavor', 'consumer.mjs'), + `export default ${JSON.stringify({ schemaVersion: 1, targets: { win32: { use: { config: './desktop.config.mjs', target: 'win32' } } } })};`, + ); + await assert.rejects(loadDesktopConfig(path.join(root, 'flavor', 'consumer.mjs')), /unknown scope key "product.flavor"/); fs.writeFileSync(path.join(root, 'one.spec.mjs'), ''); fs.writeFileSync(path.join(root, 'two.spec.mjs'), ''); assert.deepEqual(resolveDesktopSpecs({ kind: 'wdio', framework: 'jasmine', specs: [['*.spec.mjs']] }, 'win32', root), [ @@ -1188,11 +1225,110 @@ async function main() { logs.some((log) => log.includes('serve --agent requires an integration that supports agents; \\"fixture\\" does not.')), logs.join('\n'), ); + } else if (mode === 'lane-cli') { + // Each invocation is its own process with no inherited collector, so its run result records the lane it was given. + const configPath = path.join(root, 'desktop.config.mjs'); + fs.writeFileSync(path.join(root, 'spec.mjs'), 'export {};\n'); + fs.writeFileSync( + configPath, + [ + "import fs from 'node:fs';", + 'export default {', + ' schemaVersion: 1,', + " targets: { win32: { host: { kind: 'installed', application: { executablePath: 'not-launched.exe' } } } },", + ' suites: {', + " csf: { kind: 'csf-wdio' },", + " jasmine: { kind: 'wdio', framework: 'jasmine', specs: ['spec.mjs'] },", + " mocha: { kind: 'wdio', framework: 'mocha', specs: ['spec.mjs'] },", + ' },', + ' reporting: { artifactsRoot: process.env.FIXTURE_ARTIFACTS },', + ' integration: {', + " name: 'fixture',", + ' validate() {},', + ' async run({ action, suite, input, supervisor }) {', + ' fs.writeFileSync(process.env.FIXTURE_MARKER, JSON.stringify({ action, suite, input: input ?? null }));', + " supervisor.reporter.emit({ kind: 'test-result', payload: { testId: 'fixture', title: 'fixture', attempt: 0, status: 'passed', durationMs: 1 } });", + ' },', + ' },', + '};', + '', + ].join('\n'), + ); + const inherited = { ...process.env }; + for (const name of ['DESKTOP_REPORT_CONTEXT', 'DESKTOP_INPUT', 'DESKTOP_REPORTER', 'DESKTOP_EVENTS']) delete inherited[name]; + let invocations = 0; + const invoke = (args, environment = {}) => { + const index = invocations++; + const marker = path.join(root, `marker-${index}.json`); + const artifacts = path.join(root, `artifacts-${index}`); + const child = spawnSync( + process.execPath, + [path.join(packageRoot, 'config', 'cli.cjs'), '--config', configPath, '--target', 'win32', '--reporter', 'json', ...args], + { cwd: root, encoding: 'utf8', env: { ...inherited, FIXTURE_MARKER: marker, FIXTURE_ARTIFACTS: artifacts, ...environment } }, + ); + const runs = fs.existsSync(artifacts) ? fs.readdirSync(artifacts) : []; + return { + status: child.status, + diagnostics: `${child.stdout}\n${child.stderr}`, + marker: fs.existsSync(marker) ? JSON.parse(fs.readFileSync(marker, 'utf8')) : undefined, + result: runs.length ? JSON.parse(fs.readFileSync(path.join(artifacts, runs[0], 'run.json'), 'utf8')) : undefined, + error: child.stdout.startsWith('{') ? JSON.parse(child.stdout.split('\n')[0]).error : undefined, + }; + }; + const ran = (run, input, lane) => { + assert.equal(run.status, 0, run.diagnostics); + assert.deepEqual(run.marker, { action: 'test', suite: 'csf', input }); + assert.deepEqual(run.result.lane, lane); + }; + ran(invoke(['test', '--suite', 'csf', '--input', 'accessible']), 'accessible', { + input: 'accessible', + source: 'option', + masking: 'host-masked', + policyEnforced: false, + }); + ran(invoke(['test', '--suite', 'csf'], { DESKTOP_INPUT: 'physical' }), 'physical', { + input: 'physical', + source: 'environment', + policyEnforced: false, + }); + ran(invoke(['test', '--suite', 'csf', '--input', 'accessible'], { DESKTOP_INPUT: 'physical' }), 'accessible', { + input: 'accessible', + source: 'option', + masking: 'host-masked', + policyEnforced: false, + }); + const unlaned = invoke(['test', '--suite', 'csf']); + assert.equal(unlaned.status, 0, unlaned.diagnostics); + assert.deepEqual(unlaned.marker, { action: 'test', suite: 'csf', input: null }); + assert.equal('lane' in unlaned.result, false, 'a run with no lane must not claim one'); + const refused = (run, pattern) => { + assert.equal(run.status, 1, run.diagnostics); + assert.equal(run.marker, undefined, 'a refused lane must not start the run'); + assert.equal(run.result, undefined, 'a refused lane must not open a result'); + assert.equal(run.error?.code, 'config', run.diagnostics); + assert.match(run.error.message, pattern); + }; + refused(invoke(['test', '--suite', 'jasmine', '--list', '--input', 'accessible']), /--list resolves spec files/); + refused( + invoke(['test', '--suite', 'mocha', '--input', 'accessible']), + /--input accessible requires a jasmine suite; "mocha" uses mocha/, + ); + refused(invoke(['test', '--suite', 'mocha'], { DESKTOP_INPUT: 'physical' }), /--input physical requires a jasmine suite/); + refused(invoke(['test', '--suite', 'csf'], { DESKTOP_INPUT: 'fast' }), /DESKTOP_INPUT must be accessible or physical/); + const listed = invoke(['test', '--suite', 'jasmine', '--list'], { DESKTOP_INPUT: 'accessible' }); + assert.equal(listed.status, 0, `--list resolves files and ignores the environment lane\n${listed.diagnostics}`); } else { const { DesktopSupervisor } = await import(lib('supervisor.js')); const { runDesktopWdioSuite } = await import(lib('wdio/run.js')); + const { createDesktopInputLane } = await import(lib('reporting/lane.js')); + const { resolveDesktopScopeDomains } = await import('@fluentui-react-native/desktop-driver/authoring'); const { FakeDesktopHost } = await import('@fluentui-react-native/desktop-driver/testing'); - const run = reporter({ context: { endpoint: 'win32', renderer: 'paper' } }); + const laneMode = mode.startsWith('lane-'); + const retried = mode === 'retry' || mode === 'lane-physical'; + const lane = laneMode ? createDesktopInputLane(mode === 'lane-physical' ? 'physical' : 'accessible', 'option') : undefined; + // The lane fixtures import the Jasmine adapter by file URL from the environment, so no value is spliced into source. + if (laneMode) process.env.DESKTOP_JASMINE_URL = lib('jasmine/index.js'); + const run = reporter({ context: { endpoint: 'win32', renderer: 'paper' }, ...(lane ? { lane } : {}) }); const fixture = path.join(root, mode === 'typescript' ? 'spec.ts' : 'spec.mjs'); const second = path.join(root, 'second.mjs'); let source; @@ -1213,6 +1349,51 @@ async function main() { "describe('hook fixture',()=>{beforeAll(()=>{throw new Error('original setup failure');});it('does not execute',()=>{throw new Error('body must not run');});});"; } else if (mode === 'crash') { source = 'process.exit(17);'; + } else if (mode === 'lane-physical') { + source = [ + "import fs from 'node:fs';", + 'const { desktopIt, desktopDescribe } = await import(process.env.DESKTOP_JASMINE_URL);', + "const marker = new URL('./attempt', import.meta.url);", + 'const mustNotRun = (what) => () => { throw new Error(`${what} must not run`); };', + "describe('physical lane fixture', () => {", + " beforeAll(() => { globalThis.sharedHook = 'ran'; });", + " desktopIt('runs when nothing is declared', () => { expect(globalThis.sharedHook).toBe('ran'); });", + " desktopIt('clicks as the lane does', { requires: ['click'] }, async () => { await (await browser.$('~button-primary')).click(); });", + " desktopIt('needs the keyboard', { requires: ['keyboard'] }, () => {});", + " desktopIt('fails once, then retries', () => { if (!fs.existsSync(marker)) { fs.writeFileSync(marker, 'attempt'); throw new Error('first attempt'); } });", + " desktopIt('accessible only', { requires: ['accessibility-click'] }, mustNotRun('an input-excluded case'));", + " desktopIt('macOS only', { platforms: ['macos'] }, mustNotRun('a platform-excluded case'));", + " desktopIt('fabric only', { scope: { 'rn.renderer': ['fabric'] } }, mustNotRun('a scope-excluded case'));", + " desktopIt('declared gap', { unsupported: { platforms: ['win32'], reason: 'Declared gap.' } }, mustNotRun('a declared gap'));", + '});', + "desktopDescribe('excluded group', { platforms: ['macos'] }, () => {", + " beforeAll(mustNotRun('the hook of an excluded group'));", + " beforeEach(mustNotRun('the hook of an excluded group'));", + " desktopIt('inside an excluded group', mustNotRun('a platform-excluded case'));", + '});', + ].join('\n'); + } else if (mode === 'lane-accessible') { + source = [ + 'const { desktopIt } = await import(process.env.DESKTOP_JASMINE_URL);', + 'const mustNotRun = (what) => () => { throw new Error(`${what} must not run`); };', + "describe('accessible lane fixture', () => {", + " desktopIt('runs when nothing is declared', () => {});", + " desktopIt('clicks as the lane does', { requires: ['click'] }, async () => { await (await browser.$('~button-primary')).click(); });", + " desktopIt('clicks accessibly', { requires: ['accessibility-click'] }, async () => { await (await browser.$('~button-primary')).click(); });", + " desktopIt('physical only', { requires: ['keyboard'] }, mustNotRun('an input-excluded case'));", + " desktopIt('needs what the host lacks', { requires: ['accessibility-actions'] }, mustNotRun('a case the host cannot support'));", + '});', + ].join('\n'); + } else if (mode === 'lane-blocked') { + source = [ + "import fs from 'node:fs';", + 'const { desktopIt } = await import(process.env.DESKTOP_JASMINE_URL);', + "const ran = new URL('./ran', import.meta.url);", + "describe('blocked lane fixture', () => {", + " desktopIt('declared', () => { fs.writeFileSync(ran, 'ran'); });", + " it('undeclared', () => { fs.writeFileSync(ran, 'ran'); });", + '});', + ].join('\n'); } else if (mode === 'bail') { source = "describe('bail fixture',()=>{it('fails first',()=>{throw new Error('original bail failure');});});"; fs.writeFileSync( @@ -1236,7 +1417,11 @@ async function main() { projectRoot: packageRoot, configPath: path.join(root, 'desktop.config.mjs'), }; - const host = new FakeDesktopHost({ endpoint: 'win32' }); + // The accessible lane masks keyboard, physical click and wheel on the host. Only `lane-blocked` leaves them on. + const host = new FakeDesktopHost({ + endpoint: 'win32', + ...(mode === 'lane-accessible' ? { features: { keyboard: false, physicalClick: false, wheel: false } } : {}), + }); let launches = 0; const launch = host.launch.bind(host); host.launch = (...args) => { @@ -1259,7 +1444,7 @@ async function main() { kind: 'wdio', framework: 'jasmine', specs: mode === 'bail' ? [fixture, second] : [[fixture, ...(mode === 'wdio' ? [second] : [])]], - specFileRetries: mode === 'retry' ? 1 : 0, + specFileRetries: retried ? 1 : 0, }, }, }, @@ -1267,6 +1452,7 @@ async function main() { configPath: target.configPath, targets: { win32: target }, reporting: { artifactsRoot: root }, + testScopes: resolveDesktopScopeDomains(), }; if (mode === 'config-hook-failure') config.config.suites.fixture.readiness = { kind: 'native', testId: 'missing-readiness-marker', timeoutMs: 50 }; @@ -1294,7 +1480,7 @@ async function main() { }); await assert.rejects(duplicate.start(), /Another desktop workload/); try { - await runDesktopWdioSuite(supervisor, config, 'fixture'); + await runDesktopWdioSuite(supervisor, config, 'fixture', lane?.input); } catch (error) { failure = error; run.fail(error); @@ -1303,7 +1489,7 @@ async function main() { await supervisor.close(); } const result = await run.finish(); - if (['hook-failure', 'config-hook-failure', 'config-load-failure', 'crash', 'bail'].includes(mode)) { + if (['hook-failure', 'config-hook-failure', 'config-load-failure', 'crash', 'bail', 'lane-blocked'].includes(mode)) { assert(failure, `hook failure must fail launcher\n${JSON.stringify(result)}\n${err.text()}\n${out.text()}`); assert.equal(result.status, 'failed'); assert(!out.text().includes('PASSED WDIO fixture'), out.text()); @@ -1322,15 +1508,49 @@ async function main() { result.work.some(({ status }) => status === 'not-run'), JSON.stringify(result), ); + if (mode === 'lane-blocked') { + // An unmasked accessible host and an undeclared spec are both reported, and neither spec ran. + const text = JSON.stringify(result); + assert(text.includes('needs a host that reports keyboard, physicalClick, wheel disabled'), text); + assert(text.includes('not declared with desktopIt or desktopDescribe'), text); + assert(!fs.existsSync(path.join(root, 'ran')), 'a blocked lane must not run any spec'); + assert.equal(result.counts.executed, 0, text); + assert.equal(result.counts.skipped, 0, 'a blocked lane reports its fault, not every spec it disabled'); + } + } else if (mode === 'lane-accessible') { + const text = JSON.stringify(result); + // The host cannot support one case. A lane run holds that against the run instead of quietly skipping it. + assert(failure && /incomplete/.test(failure.message), `${failure?.stack}\n${text}\n${err.text()}`); + assert.notEqual(result.status, 'passed', text); + assert.equal(launches, 1); + assert.equal(result.counts.passed, 3, text); + assert.equal(result.counts.failed, 0, text); + assert.equal(result.counts.unsupported, 1, text); + assert.equal(result.counts.declaredUnsupported, 0, text); + assert.deepEqual(result.selection.excluded, { platform: 0, scope: 0, input: 1 }, text); + assert.deepEqual(result.lane, { input: 'accessible', source: 'option', masking: 'host-masked', policyEnforced: false }); + const clicks = host.actions.filter((action) => action.type === 'click'); + assert(clicks.length === 2 && clicks.every((action) => action.mode === 'accessibility'), JSON.stringify(host.actions)); } else { assert.equal(failure, undefined, `${failure?.stack}\n${err.text()}\n${out.text()}`); assert.equal(result.status, 'passed', JSON.stringify(result)); - assert.equal(launches, mode === 'retry' ? 2 : 1, 'WDIO must own exactly one session per grouped attempt'); - assert.equal(result.counts.flaky, mode === 'retry' ? 1 : 0, JSON.stringify(result)); + assert.equal(launches, retried ? 2 : 1, 'WDIO must own exactly one session per grouped attempt'); + assert.equal(result.counts.flaky, retried ? 1 : 0, JSON.stringify(result)); if (mode === 'wdio') { assert.equal(result.counts.passed, 2); assert.equal(result.counts.skipped, 1); } + if (mode === 'lane-physical') { + const text = JSON.stringify(result); + assert.equal(result.counts.passed, 4, text); + assert.equal(result.counts.unsupported, 1, text); + assert.equal(result.counts.declaredUnsupported, 1, text); + // Both attempts announced the same exclusions, and a retried group still counts them once. + assert.deepEqual(result.selection.excluded, { platform: 2, scope: 1, input: 1 }, text); + assert.deepEqual(result.lane, { input: 'physical', source: 'option', policyEnforced: false }); + const clicks = host.actions.filter((action) => action.type === 'click'); + assert(clicks.length > 0 && clicks.every((action) => action.mode === 'physical'), JSON.stringify(host.actions)); + } } assert(!fs.existsSync(path.join(root, 'locks', 'interactive.lock'))); } diff --git a/packages/agentic/desktop-test/src/contracts.test.ts b/packages/agentic/desktop-test/src/contracts.test.ts index 559708c809..d666a3dcd5 100644 --- a/packages/agentic/desktop-test/src/contracts.test.ts +++ b/packages/agentic/desktop-test/src/contracts.test.ts @@ -25,6 +25,10 @@ describe('neutral desktop harness contracts', () => { 'config-load-failure', 'crash', 'bail', + 'lane-physical', + 'lane-accessible', + 'lane-blocked', + 'lane-cli', ])( '%s contract', async (mode) => { diff --git a/packages/agentic/desktop-test/src/index.ts b/packages/agentic/desktop-test/src/index.ts index 946718dc9b..1b61b3ad75 100644 --- a/packages/agentic/desktop-test/src/index.ts +++ b/packages/agentic/desktop-test/src/index.ts @@ -10,7 +10,7 @@ export type { PublishDesktopAgentConnection, } from './agentConnection.js'; export { DesktopServiceScope, DesktopStopRequested } from './services.js'; -export { defineDesktopConfig, loadDesktopConfig, resolveDesktopSpecs, validateDesktopConfig } from './config.js'; +export { defineDesktopConfig, loadDesktopConfig, resolveDesktopSpecs, resolveDesktopTestScope, validateDesktopConfig } from './config.js'; export type { DesktopConfig, DesktopTargetConfig, DesktopWdioSuite, ResolvedDesktopConfig, ResolvedDesktopTarget } from './config.js'; export type { DesktopIntegration } from './integration.js'; export { desktopStageCommands, resolveDesktopApplication } from './adapters.js'; diff --git a/packages/agentic/desktop-test/src/integration.ts b/packages/agentic/desktop-test/src/integration.ts index cdb74ede46..35fc5021a6 100644 --- a/packages/agentic/desktop-test/src/integration.ts +++ b/packages/agentic/desktop-test/src/integration.ts @@ -1,4 +1,5 @@ import type { DesktopHost, DesktopTarget } from '@fluentui-react-native/desktop-driver'; +import type { DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; import type { ResolvedDesktopReporterMode } from '@fluentui-react-native/desktop-driver/cli/reporter'; import type { DesktopAgentConnectionOptions } from './agentConnection.js'; import type { DesktopStage, ResolvedDesktopConfig, ResolvedDesktopTarget } from './config.js'; @@ -30,6 +31,8 @@ export type DesktopIntegration = { supervisor: DesktopSupervisor; action: DesktopStage | 'test' | 'serve'; suite?: string; + /** Set only for `test`, when the run executes one input lane. The integration owns masking its own driver. */ + input?: DesktopInputLane; /** Set only for `serve --agent`, and only for an integration that `supportsAgent`. */ agent?: Pick; /** diff --git a/packages/agentic/desktop-test/src/jasmine/adapter.test.ts b/packages/agentic/desktop-test/src/jasmine/adapter.test.ts new file mode 100644 index 0000000000..5b0764a76c --- /dev/null +++ b/packages/agentic/desktop-test/src/jasmine/adapter.test.ts @@ -0,0 +1,208 @@ +import { createDesktopJasmine } from './adapter.js'; +import type { DesktopJasmineApi } from './adapter.js'; +import { createDesktopJasmineRegistry, desktopSpecIdentifier, excludedSpecCounts } from './registry.js'; +import { desktopTestSelectionEnvironment, serializeDesktopTestSelection } from './selection.js'; + +type DesktopBody = (done?: (error?: unknown) => void) => unknown; +type Registered = { kind: 'describe' | 'it' | 'xit'; name: string; path: string[]; body?: DesktopBody }; + +/** Registers what a Jasmine would, and hands back the metadata `it` and `xit` return. */ +function fakeJasmine(options: { identify?: boolean } = {}) { + const registered: Registered[] = []; + const path: string[] = []; + let next = 0; + const spec = (kind: 'it' | 'xit', name: string, body?: DesktopBody) => { + registered.push({ kind, name, path: [...path], body }); + return options.identify === false ? undefined : { id: `spec${next++}`, description: name }; + }; + const api: DesktopJasmineApi = { + describe(name, body) { + registered.push({ kind: 'describe', name, path: [...path] }); + path.push(name); + try { + body.call({}); + } finally { + path.pop(); + } + }, + it: (name, body) => spec('it', name, body), + xit: (name) => spec('xit', name), + }; + return { api, registered }; +} + +function laneEnvironment(lane?: 'accessible' | 'physical', extra: { testScopes?: unknown; values?: Record } = {}) { + return { + [desktopTestSelectionEnvironment]: serializeDesktopTestSelection({ + endpoint: 'win32', + ...(lane ? { lane } : {}), + ...(extra.testScopes ? { testScopes: extra.testScopes } : {}), + values: { 'rn.renderer': 'paper', ...extra.values }, + }), + }; +} + +function setup(lane?: 'accessible' | 'physical', extra?: Parameters[1], options?: { identify?: boolean }) { + const registry = createDesktopJasmineRegistry(laneEnvironment(lane, extra)); + const jasmine = fakeJasmine(options); + return { registry, ...jasmine, ...createDesktopJasmine(jasmine.api, registry) }; +} + +const body = () => undefined; + +describe('desktopIt without a selection', () => { + test('registers an ordinary spec, so a runner that does not select keeps working', () => { + const registry = createDesktopJasmineRegistry(); + const { api, registered } = fakeJasmine(); + const { desktopIt } = createDesktopJasmine(api, registry); + desktopIt('plain', body); + desktopIt('physical', { requires: ['keyboard'] }, body); + + expect(registered.map(({ kind, name }) => `${kind}:${name}`)).toEqual(['it:plain', 'it:physical']); + expect(registry.errors).toEqual([]); + }); + + test('still rejects a declaration it cannot read', () => { + const registry = createDesktopJasmineRegistry(); + const { api, registered } = fakeJasmine(); + const { desktopIt } = createDesktopJasmine(api, registry); + desktopIt('bad', { requires: ['telepathy' as never] }, body); + + expect(registered).toHaveLength(1); + expect(() => registered[0].body?.()).toThrow(/unsupported requirement/); + expect(registry.entries.get('badspec0')).toEqual({ kind: 'invalid' }); + }); +}); + +describe('desktopIt in a lane', () => { + test('runs what the lane covers and replaces the rest with placeholders that are counted by reason', () => { + const { desktopIt, registered, registry } = setup('accessible'); + desktopIt('both', body); + desktopIt('accessible only', { requires: ['accessibility-click'] }, body); + desktopIt('physical only', { requires: ['keyboard'] }, body); + desktopIt('macOS only', { platforms: ['macos'] }, body); + desktopIt('fabric only', { scope: { 'rn.renderer': ['fabric'] } }, body); + + expect(registered.map(({ kind, name }) => `${kind}:${name}`)).toEqual([ + 'it:both', + 'it:accessible only', + 'xit:physical only', + 'xit:macOS only', + 'xit:fabric only', + ]); + expect(excludedSpecCounts(registry)).toEqual({ platform: 1, scope: 1, input: 1 }); + expect(registry.entries.get('physical onlyspec2')).toEqual({ kind: 'excluded', reason: 'input' }); + }); + + test('a case is excluded once, for the first reason that applies', () => { + const { desktopIt, registry } = setup('accessible'); + desktopIt('everything', { platforms: ['macos'], scope: { 'rn.renderer': ['fabric'] }, requires: ['keyboard'] }, body); + + expect(excludedSpecCounts(registry)).toEqual({ platform: 1, scope: 0, input: 0 }); + }); + + test('a declared gap becomes a placeholder that carries its reason, not an excluded case', () => { + const { desktopIt, registered, registry } = setup('physical'); + desktopIt('known gap', { unsupported: { platforms: ['win32'], reason: 'Not on win32 yet.' } }, body); + desktopIt('known gap elsewhere', { unsupported: { platforms: ['macos'], reason: 'Not on macOS yet.' } }, body); + + expect(registered.map(({ kind }) => kind)).toEqual(['xit', 'it']); + expect(registry.entries.get('known gapspec0')).toEqual({ kind: 'gap', gap: { kind: 'unsupported', reason: 'Not on win32 yet.' } }); + expect(excludedSpecCounts(registry)).toEqual({ platform: 0, scope: 0, input: 0 }); + }); + + test('an expired quarantine fails that case instead of hiding it', () => { + const { desktopIt, registered, registry } = setup('physical'); + desktopIt('stale', { quarantine: { issue: '#1', reason: 'Flaky.', expires: '2000-01-01' } }, body); + + expect(registered.map(({ kind }) => kind)).toEqual(['it']); + expect(() => registered[0].body?.()).toThrow('Quarantine #1 expired on 2000-01-01.'); + expect(registry.entries.get('stalespec0')).toEqual({ kind: 'invalid' }); + }); + + test('a custom scope key is checked against the domains the config declared', () => { + const extra = { testScopes: { flavor: ['dev', 'prod'] }, values: { flavor: 'dev' } }; + const accepted = setup('physical', extra); + accepted.desktopIt('dev only', { scope: { flavor: ['dev'] } }, body); + accepted.desktopIt('prod only', { scope: { flavor: ['prod'] } }, body); + expect(accepted.registered.map(({ kind }) => kind)).toEqual(['it', 'xit']); + + const unknown = setup('physical'); + unknown.desktopIt('typo', { scope: { flavour: ['dev'] } }, body); + expect(() => unknown.registered[0].body?.()).toThrow(/unknown scope key "flavour"/); + }); + + test('records the spec by the identifier WDIO reports, and faults when Jasmine does not identify it', () => { + expect(desktopSpecIdentifier({ description: 'a case', id: 'spec7' })).toBe('a casespec7'); + expect(desktopSpecIdentifier({ description: 'a case' })).toBeUndefined(); + + const { desktopIt, registry } = setup('physical', undefined, { identify: false }); + desktopIt('anonymous', body); + expect(registry.errors).toEqual([expect.stringContaining('did not identify the spec registered for "anonymous"')]); + }); +}); + +describe('desktopDescribe', () => { + test('adds what a group requires to every case inside it, however deep', () => { + const { desktopDescribe, desktopIt, registered } = setup('accessible'); + desktopDescribe('group', { requires: ['keyboard'] }, () => { + desktopIt('inherits', body); + desktopDescribe('nested', () => desktopIt('also inherits', body)); + }); + desktopIt('outside', body); + + expect(registered.filter(({ kind }) => kind !== 'describe').map(({ kind, name }) => `${kind}:${name}`)).toEqual([ + 'xit:inherits', + 'xit:also inherits', + 'it:outside', + ]); + }); + + test('narrows platforms to the overlap of the group and the case', () => { + const { desktopDescribe, desktopIt, registered } = setup('physical'); + desktopDescribe('group', { platforms: ['!macos'] }, () => { + desktopIt('keeps win32', { platforms: ['macos', 'win32'] }, body); + desktopIt('loses win32', { platforms: ['macos', 'windows'] }, body); + }); + + expect(registered.filter(({ kind }) => kind !== 'describe').map(({ kind, name }) => `${kind}:${name}`)).toEqual([ + 'it:keeps win32', + 'xit:loses win32', + ]); + }); + + test('a case that could never run inside its group is a failing spec, not a silent exclusion', () => { + const { desktopDescribe, desktopIt, registered } = setup('physical'); + desktopDescribe('group', { platforms: ['macos'] }, () => { + desktopIt('never', { platforms: ['win32'] }, body); + }); + + const specs = registered.filter(({ kind }) => kind === 'it'); + expect(specs).toHaveLength(1); + expect(() => specs[0].body?.()).toThrow(/do not overlap the enclosing group's, so it could never run/); + }); + + test('a group cannot declare a gap, so its body does not run and one spec reports why', () => { + const { desktopDescribe, registered } = setup('physical'); + const inner = jest.fn(); + desktopDescribe('group', { unsupported: { platforms: ['win32'], reason: 'No.' } } as never, inner); + + expect(inner).not.toHaveBeenCalled(); + const specs = registered.filter(({ kind }) => kind === 'it'); + expect(specs).toHaveLength(1); + expect(() => specs[0].body?.()).toThrow(/cannot declare unsupported or quarantine/); + }); + + test('a group that throws still releases its options for the next group', () => { + const { desktopDescribe, desktopIt, registered, registry } = setup('accessible'); + expect(() => + desktopDescribe('throws', { requires: ['keyboard'] }, () => { + throw new Error('boom'); + }), + ).toThrow('boom'); + expect(registry.groups).toEqual([]); + + desktopIt('after', body); + expect(registered.filter(({ kind }) => kind === 'it' || kind === 'xit').map(({ kind }) => kind)).toEqual(['it']); + }); +}); diff --git a/packages/agentic/desktop-test/src/jasmine/adapter.ts b/packages/agentic/desktop-test/src/jasmine/adapter.ts new file mode 100644 index 0000000000..9139684e15 --- /dev/null +++ b/packages/agentic/desktop-test/src/jasmine/adapter.ts @@ -0,0 +1,103 @@ +import { + decideDesktopCase, + mergeDesktopCaseOptions, + validateDesktopCaseOptions, + validateDesktopGroupOptions, +} from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopCaseOptions, ResolvedDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; +import { desktopSpecIdentifier } from './registry.js'; +import type { DesktopJasmineRegistry, DesktopSpecEntry } from './registry.js'; + +export type DesktopSpecBody = (done: (error?: unknown) => void) => unknown; +export type DesktopGroupBody = (this: unknown) => void; +/** A group scopes the cases inside it. A gap describes one case, so a group cannot declare one. */ +export type DesktopGroupOptions = Omit; + +/** The part of the Jasmine globals the adapter registers specs through. */ +export type DesktopJasmineApi = { + describe(name: string, body: DesktopGroupBody): unknown; + it(name: string, body: DesktopSpecBody): unknown; + xit(name: string): unknown; +}; + +/** + * Declares tests with their input modality and the places they run, and registers them with Jasmine. + * + * `desktopIt` and `desktopDescribe` never throw for a bad declaration: Jasmine only warns when a spec file fails to load, + * so a throw would silently drop every test after it. A bad declaration becomes a failing spec that reports the problem. + */ +export function createDesktopJasmine(api: DesktopJasmineApi, registry: DesktopJasmineRegistry) { + function record(spec: unknown, name: string, entry: DesktopSpecEntry): void { + const identifier = desktopSpecIdentifier(spec as { description?: unknown; id?: unknown } | undefined); + if (identifier === undefined) { + registry.errors.push(`Jasmine did not identify the spec registered for "${name}", so its selection cannot be verified.`); + return; + } + registry.entries.set(identifier, entry); + } + + function failing(name: string, error: unknown): void { + record( + api.it(name, () => { + throw error; + }), + name, + { kind: 'invalid' }, + ); + } + + function desktopIt(name: string, optionsOrBody: DesktopCaseOptions | DesktopSpecBody, maybeBody?: DesktopSpecBody): void { + const hasOptions = typeof optionsOrBody !== 'function'; + const body = hasOptions ? maybeBody : optionsOrBody; + const title = typeof name === 'string' && name.trim() ? name : 'desktopIt'; + const source = `desktopIt("${title}")`; + let descriptor: ResolvedDesktopCaseOptions; + let decision: ReturnType; + try { + if (title !== name) throw new TypeError('desktopIt requires a non-empty name.'); + if (typeof body !== 'function') throw new TypeError(`${source} requires a test function.`); + const declared = validateDesktopCaseOptions(hasOptions ? optionsOrBody : {}, `${source} options`); + descriptor = mergeDesktopCaseOptions(registry.groups.at(-1), declared, source); + decision = registry.selection ? decideDesktopCase(descriptor, registry.selection, source) : { kind: 'run' }; + } catch (error) { + failing(title, error); + return; + } + if (decision.kind === 'run') { + record(api.it(name, body), name, { kind: 'run', descriptor }); + return; + } + // A placeholder keeps the spec out of Jasmine's hooks and body without reporting it as a skipped test. + const entry: DesktopSpecEntry = + decision.kind === 'excluded' ? { kind: 'excluded', reason: decision.reason } : { kind: 'gap', gap: decision.gap }; + record(api.xit(name), name, entry); + } + + function desktopDescribe(name: string, optionsOrBody: DesktopGroupOptions | DesktopGroupBody, maybeBody?: DesktopGroupBody): void { + const hasOptions = typeof optionsOrBody !== 'function'; + const body = hasOptions ? maybeBody : optionsOrBody; + const title = typeof name === 'string' && name.trim() ? name : 'desktopDescribe'; + const source = `desktopDescribe("${title}")`; + let group: ResolvedDesktopCaseOptions; + try { + if (title !== name) throw new TypeError('desktopDescribe requires a non-empty name.'); + if (typeof body !== 'function') throw new TypeError(`${source} requires a body.`); + const declared = validateDesktopGroupOptions(hasOptions ? optionsOrBody : {}, `${source} options`); + group = mergeDesktopCaseOptions(registry.groups.at(-1), declared, source); + } catch (error) { + // The body does not run, so nothing inside an unreadable group is selected by accident. + api.describe(title, () => failing('has invalid desktop options', error)); + return; + } + api.describe(name, function (this: unknown) { + registry.groups.push(group); + try { + body.call(this); + } finally { + registry.groups.pop(); + } + }); + } + + return { desktopIt, desktopDescribe }; +} diff --git a/packages/agentic/desktop-test/src/jasmine/index.ts b/packages/agentic/desktop-test/src/jasmine/index.ts new file mode 100644 index 0000000000..b602b7c452 --- /dev/null +++ b/packages/agentic/desktop-test/src/jasmine/index.ts @@ -0,0 +1,42 @@ +import { createDesktopJasmine } from './adapter.js'; +import type { DesktopGroupBody, DesktopGroupOptions, DesktopJasmineApi, DesktopSpecBody } from './adapter.js'; +import { desktopJasmineRegistry } from './registry.js'; +import type { DesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; + +export type { + DesktopCaseOptions, + DesktopInput, + DesktopPlatforms, + DesktopRequirement, +} from '@fluentui-react-native/desktop-driver/authoring'; +export type { DesktopGroupBody, DesktopGroupOptions, DesktopSpecBody } from './adapter.js'; + +// The Jasmine globals exist only once a framework has installed them, so they are read when a declaration runs. +const globals = () => globalThis as unknown as DesktopJasmineApi; +const jasmineApi: DesktopJasmineApi = { + describe: (name, body) => globals().describe(name, body), + it: (name, body) => globals().it(name, body), + xit: (name) => globals().xit(name), +}; +const adapter = () => createDesktopJasmine(jasmineApi, desktopJasmineRegistry()); + +/** + * Declares a Jasmine test with the input it needs and where it runs. The descriptor decides whether the test is part of a + * run: `requires` implies the input modality, `platforms` and `scope` narrow where it runs, and `unsupported` or + * `quarantine` report a known gap instead of executing. + */ +export function desktopIt(name: string, body: DesktopSpecBody): void; +export function desktopIt(name: string, options: DesktopCaseOptions, body: DesktopSpecBody): void; +export function desktopIt(name: string, optionsOrBody: DesktopCaseOptions | DesktopSpecBody, maybeBody?: DesktopSpecBody): void { + adapter().desktopIt(name, optionsOrBody, maybeBody); +} + +/** + * Declares a Jasmine group whose options apply to every test inside it: requirements add up, platforms and scope intersect, + * and a nested declaration can only narrow its group. + */ +export function desktopDescribe(name: string, body: DesktopGroupBody): void; +export function desktopDescribe(name: string, options: DesktopGroupOptions, body: DesktopGroupBody): void; +export function desktopDescribe(name: string, optionsOrBody: DesktopGroupOptions | DesktopGroupBody, maybeBody?: DesktopGroupBody): void { + adapter().desktopDescribe(name, optionsOrBody, maybeBody); +} diff --git a/packages/agentic/desktop-test/src/jasmine/registry.ts b/packages/agentic/desktop-test/src/jasmine/registry.ts new file mode 100644 index 0000000000..d927d700f1 --- /dev/null +++ b/packages/agentic/desktop-test/src/jasmine/registry.ts @@ -0,0 +1,72 @@ +import type { + DesktopDeclaredGap, + DesktopExclusionReason, + DesktopRequirement, + ResolvedDesktopCaseOptions, +} from '@fluentui-react-native/desktop-driver/authoring'; +import type { SelectionExclusions } from '../reporting/protocol.js'; +import { desktopTestSelectionEnvironment, parseDesktopTestSelection } from './selection.js'; +import type { DesktopTestSelection } from './selection.js'; + +/** What the adapter decided for a spec it registered. */ +export type DesktopSpecEntry = + /** Executes, unless a preflight found that the host cannot meet what it requires. */ + | { kind: 'run'; descriptor: ResolvedDesktopCaseOptions; missing?: readonly DesktopRequirement[] } + /** Registered as a placeholder that never runs and is counted, not reported as a test. */ + | { kind: 'excluded'; reason: DesktopExclusionReason } + /** Registered as a placeholder that is reported as unsupported, with the reason its author gave. */ + | { kind: 'gap'; gap: DesktopDeclaredGap } + /** A failing spec the adapter registered in place of a declaration it could not accept. */ + | { kind: 'invalid' }; + +/** What one worker process learned while its spec files registered their tests. */ +export type DesktopJasmineRegistry = { + /** Absent when no desktop-test run started this process, in which case declarations are only validated. */ + readonly selection: DesktopTestSelection | undefined; + /** Keyed by the identifier WDIO reports for a spec: its description followed by its Jasmine id. */ + readonly entries: Map; + /** The enclosing `desktopDescribe` options while a group body registers, innermost last. */ + readonly groups: ResolvedDesktopCaseOptions[]; + /** Faults that make the selection untrustworthy, such as a spec the adapter could not identify. */ + readonly errors: string[]; + /** Set once every spec was disabled because of a fault, so the reporter shows the fault instead of a wall of skips. */ + blocked: boolean; +}; + +const registryKey = Symbol.for('@fluentui-react-native/desktop-test/jasmine-registry'); + +/** + * The registry of this process. It lives on `globalThis` because the spec files and the reporter share one worker process + * but may load this module through different paths, such as source and its build. + */ +export function desktopJasmineRegistry(environment: NodeJS.ProcessEnv = process.env): DesktopJasmineRegistry { + const holder = globalThis as { [registryKey]?: DesktopJasmineRegistry }; + return (holder[registryKey] ??= createDesktopJasmineRegistry(environment)); +} + +export function createDesktopJasmineRegistry(environment: NodeJS.ProcessEnv = {}): DesktopJasmineRegistry { + const errors: string[] = []; + let selection: DesktopTestSelection | undefined; + const raw = environment[desktopTestSelectionEnvironment]; + if (raw !== undefined) { + try { + selection = parseDesktopTestSelection(raw); + } catch (error) { + errors.push(error instanceof Error ? error.message : String(error)); + } + } + return { selection, entries: new Map(), groups: [], errors, blocked: false }; +} + +/** The identifier WDIO's Jasmine adapter reports for a spec, so the reporter can find what the adapter decided. */ +export function desktopSpecIdentifier(spec: { description?: unknown; id?: unknown } | undefined): string | undefined { + return typeof spec?.description === 'string' && typeof spec.id === 'string' ? `${spec.description}${spec.id}` : undefined; +} + +export function excludedSpecCounts(registry: DesktopJasmineRegistry): SelectionExclusions { + const excluded = { platform: 0, scope: 0, input: 0 }; + for (const entry of registry.entries.values()) { + if (entry.kind === 'excluded') excluded[entry.reason]++; + } + return excluded; +} diff --git a/packages/agentic/desktop-test/src/jasmine/selection.test.ts b/packages/agentic/desktop-test/src/jasmine/selection.test.ts new file mode 100644 index 0000000000..83edda6692 --- /dev/null +++ b/packages/agentic/desktop-test/src/jasmine/selection.test.ts @@ -0,0 +1,50 @@ +import { + desktopTestSelectionEnvironment, + desktopTestSelectionFromEnvironment, + parseDesktopTestSelection, + serializeDesktopTestSelection, +} from './selection.js'; + +describe('desktop test selection', () => { + test('round-trips a lane, the custom scope domains and the target values', () => { + const raw = serializeDesktopTestSelection({ + endpoint: 'macos', + lane: 'accessible', + testScopes: { flavor: ['dev', 'prod'] }, + values: { 'rn.renderer': 'fabric', flavor: 'dev' }, + }); + + const selection = parseDesktopTestSelection(raw); + + expect(selection).toMatchObject({ endpoint: 'macos', lane: 'accessible', values: { 'rn.renderer': 'fabric', flavor: 'dev' } }); + expect(selection.domains.flavor).toEqual(['dev', 'prod']); + expect(selection.domains['rn.renderer']).toEqual(['fabric', 'paper']); + }); + + test('no lane means a run that only applies platform, scope and declared gaps', () => { + const selection = parseDesktopTestSelection(serializeDesktopTestSelection({ endpoint: 'win32', values: {} })); + + expect(selection.lane).toBeUndefined(); + }); + + test('is absent when no run started the process, and malformed values fail loudly rather than being guessed at', () => { + expect(desktopTestSelectionFromEnvironment({})).toBeUndefined(); + expect(desktopTestSelectionFromEnvironment({ [desktopTestSelectionEnvironment]: '{"endpoint":"win32","values":{}}' })).toMatchObject({ + endpoint: 'win32', + }); + expect(() => desktopTestSelectionFromEnvironment({ [desktopTestSelectionEnvironment]: '' })).toThrow('is not valid JSON'); + }); + + test.each([ + ['not json', 'is not valid JSON'], + ['[]', 'must be an object'], + ['null', 'must be an object'], + ['{"endpoint":"linux","values":{}}', 'endpoint must be one of: macos, windows, win32'], + ['{"endpoint":"win32","lane":"fast","values":{}}', 'lane must be one of: accessible, physical'], + ['{"endpoint":"win32","values":{},"extra":1}', 'unknown key "extra"'], + ['{"endpoint":"win32","values":{"flavor":"dev"}}', 'flavor'], + ['{"endpoint":"win32","testScopes":{"rn.renderer":["x"]},"values":{}}', 'rn.renderer'], + ])('rejects %s', (raw, message) => { + expect(() => parseDesktopTestSelection(raw)).toThrow(message); + }); +}); diff --git a/packages/agentic/desktop-test/src/jasmine/selection.ts b/packages/agentic/desktop-test/src/jasmine/selection.ts new file mode 100644 index 0000000000..b7e23d54db --- /dev/null +++ b/packages/agentic/desktop-test/src/jasmine/selection.ts @@ -0,0 +1,59 @@ +import type { DesktopEndpoint } from '@fluentui-react-native/desktop-driver'; +import { desktopInputLanes, resolveDesktopScopeDomains, resolveDesktopScopeValues } from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopCaseTarget, DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; + +/** Carries a run's selection from its launcher to its WDIO workers. The value is data that is parsed, never evaluated. */ +export const desktopTestSelectionEnvironment = 'DESKTOP_TEST_SELECTION'; + +/** What a worker judges each declared case against. */ +export type DesktopTestSelection = Omit; + +/** The serialized selection: the custom scope domains and the target's values, which a worker validates again. */ +export type DesktopTestSelectionRequest = { + endpoint: DesktopEndpoint; + lane?: DesktopInputLane; + testScopes?: unknown; + values: Readonly>; +}; + +const endpoints: readonly string[] = ['macos', 'windows', 'win32']; + +export function serializeDesktopTestSelection(request: DesktopTestSelectionRequest): string { + const { endpoint, lane, testScopes, values } = request; + return JSON.stringify({ endpoint, ...(lane ? { lane } : {}), ...(testScopes ? { testScopes } : {}), values }); +} + +/** Validates the selection a launcher sent. A malformed one is an error, so a worker never guesses what to run. */ +export function parseDesktopTestSelection(raw: string): DesktopTestSelection { + let value: unknown; + try { + value = JSON.parse(raw); + } catch { + throw new TypeError(`${desktopTestSelectionEnvironment} is not valid JSON.`); + } + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new TypeError(`${desktopTestSelectionEnvironment} must be an object.`); + } + const selection = value as Record; + const unknown = Object.keys(selection).find((key) => !['endpoint', 'lane', 'testScopes', 'values'].includes(key)); + if (unknown !== undefined) throw new TypeError(`${desktopTestSelectionEnvironment} has unknown key "${unknown}".`); + if (typeof selection.endpoint !== 'string' || !endpoints.includes(selection.endpoint)) { + throw new TypeError(`${desktopTestSelectionEnvironment}.endpoint must be one of: ${endpoints.join(', ')}.`); + } + if (selection.lane !== undefined && !desktopInputLanes.some((lane) => lane === selection.lane)) { + throw new TypeError(`${desktopTestSelectionEnvironment}.lane must be one of: ${desktopInputLanes.join(', ')}.`); + } + const domains = resolveDesktopScopeDomains(selection.testScopes, `${desktopTestSelectionEnvironment}.testScopes`); + return { + endpoint: selection.endpoint as DesktopEndpoint, + ...(selection.lane ? { lane: selection.lane as DesktopInputLane } : {}), + domains, + values: resolveDesktopScopeValues(selection.values ?? {}, domains, `${desktopTestSelectionEnvironment}.values`), + }; +} + +/** The selection of this process, or undefined when no desktop-test run started it. */ +export function desktopTestSelectionFromEnvironment(environment: NodeJS.ProcessEnv = process.env): DesktopTestSelection | undefined { + const raw = environment[desktopTestSelectionEnvironment]; + return raw === undefined ? undefined : parseDesktopTestSelection(raw); +} diff --git a/packages/agentic/desktop-test/src/jasmine/specSelection.test.ts b/packages/agentic/desktop-test/src/jasmine/specSelection.test.ts new file mode 100644 index 0000000000..85c105ec72 --- /dev/null +++ b/packages/agentic/desktop-test/src/jasmine/specSelection.test.ts @@ -0,0 +1,183 @@ +import { createDesktopJasmine } from './adapter.js'; +import type { DesktopJasmineApi } from './adapter.js'; +import { createDesktopJasmineRegistry } from './registry.js'; +import { desktopTestSelectionEnvironment, serializeDesktopTestSelection } from './selection.js'; +import { selectDesktopSpecs } from './specSelection.js'; +import type { DesktopJasmineEnvironment } from './specSelection.js'; + +type Node = { id?: string; description: string; children?: Node[] }; +type SpecFilter = (spec: unknown) => boolean; + +const masked = { keyboard: false, physicalClick: false, wheel: false }; +const body = () => undefined; + +/** A Jasmine-shaped suite tree that the adapter registers into, and the environment that exposes it. */ +function setup(lane?: 'accessible' | 'physical', configured?: SpecFilter) { + const registry = createDesktopJasmineRegistry({ + [desktopTestSelectionEnvironment]: serializeDesktopTestSelection({ + endpoint: 'win32', + ...(lane ? { lane } : {}), + values: { 'rn.renderer': 'paper' }, + }), + }); + const root: Node = { description: 'top suite', children: [] }; + const stack = [root]; + let next = 0; + const add = (description: string) => { + const spec = { id: `spec${next++}`, description }; + stack.at(-1)!.children!.push(spec); + return spec; + }; + const api: DesktopJasmineApi = { + describe(description, suiteBody) { + const suite: Node = { description, children: [] }; + stack.at(-1)!.children!.push(suite); + stack.push(suite); + try { + suiteBody.call({}); + } finally { + stack.pop(); + } + }, + it: (description) => add(description), + xit: (description) => add(description), + }; + let specFilter = configured; + const configure = jest.fn((configuration: { specFilter: SpecFilter }) => { + specFilter = configuration.specFilter; + }); + const environment: DesktopJasmineEnvironment = { configuration: () => ({ specFilter }), configure, topSuite: () => root }; + const spec = (description: string) => { + const found = root.children!.flatMap((child) => (child.children ?? [child]) as Node[]).find((node) => node.description === description); + if (!found) throw new Error(`No spec named ${description}.`); + return found; + }; + return { + registry, + environment, + configure, + api, + spec, + runs: (description: string) => specFilter?.(spec(description)) ?? true, + ...createDesktopJasmine(api, registry), + }; +} + +describe('selectDesktopSpecs preflight', () => { + test('excludes a spec the host cannot support, so its hooks never run, and leaves the rest', () => { + const { desktopIt, environment, registry, runs } = setup('physical'); + desktopIt('needs keyboard', { requires: ['keyboard'] }, body); + desktopIt('needs click', { requires: ['physical-click'] }, body); + desktopIt('needs nothing', body); + + selectDesktopSpecs(registry, environment, { keyboard: false, physicalClick: true }); + + expect([runs('needs keyboard'), runs('needs click'), runs('needs nothing')]).toEqual([false, true, true]); + expect(registry.entries.get('needs keyboardspec0')).toMatchObject({ kind: 'run', missing: ['keyboard'] }); + expect(registry.entries.get('needs clickspec1')).not.toHaveProperty('missing'); + }); + + test('keeps the filter the user already configured', () => { + const { desktopIt, environment, registry, runs } = setup('physical', (spec) => (spec as Node).description !== 'user filtered'); + desktopIt('user filtered', body); + desktopIt('needs keyboard', { requires: ['keyboard'] }, body); + desktopIt('kept', body); + + selectDesktopSpecs(registry, environment, { keyboard: false }); + + expect([runs('user filtered'), runs('needs keyboard'), runs('kept')]).toEqual([false, false, true]); + }); + + test('does not touch the filter when the host meets every requirement', () => { + const { configure, desktopIt, environment, registry } = setup('physical'); + desktopIt('needs keyboard', { requires: ['keyboard'] }, body); + + selectDesktopSpecs(registry, environment, { keyboard: true }); + + expect(configure).not.toHaveBeenCalled(); + }); + + test('judges a click requirement by what the lane clicks with', () => { + const physical = setup('physical'); + physical.desktopIt('clicks', { requires: ['click'] }, body); + selectDesktopSpecs(physical.registry, physical.environment, { physicalClick: false, accessibilityClick: true }); + expect(physical.runs('clicks')).toBe(false); + + const accessible = setup('accessible'); + accessible.desktopIt('clicks', { requires: ['click'] }, body); + selectDesktopSpecs(accessible.registry, accessible.environment, { ...masked, accessibilityClick: true }); + expect(accessible.runs('clicks')).toBe(true); + }); +}); + +describe('selectDesktopSpecs fail-closed checks', () => { + test('an accessible lane on a host that still offers physical input disables every spec and says why', () => { + const { desktopIt, environment, registry, runs } = setup('accessible'); + desktopIt('fine', body); + + expect(() => selectDesktopSpecs(registry, environment, { keyboard: true, physicalClick: false })).toThrow( + 'The accessible input lane needs a host that reports keyboard, wheel disabled, and this one does not.', + ); + expect(registry.blocked).toBe(true); + expect(runs('fine')).toBe(false); + }); + + test('an absent feature report counts as unmasked', () => { + const { environment, registry } = setup('accessible'); + + expect(() => selectDesktopSpecs(registry, environment, undefined)).toThrow(/keyboard, physicalClick, wheel disabled/); + }); + + test('an accessible lane on a masked host runs', () => { + const { desktopIt, environment, registry, runs } = setup('accessible'); + desktopIt('fine', body); + + selectDesktopSpecs(registry, environment, masked); + + expect(registry.blocked).toBe(false); + expect(runs('fine')).toBe(true); + }); + + test('the physical lane needs no mask', () => { + const { desktopIt, environment, registry } = setup('physical'); + desktopIt('fine', body); + + expect(() => selectDesktopSpecs(registry, environment, { keyboard: true })).not.toThrow(); + }); + + test('a lane refuses specs that were not declared with desktopIt, however many', () => { + const { api, desktopIt, environment, registry, runs } = setup('physical'); + desktopIt('declared', body); + api.describe('legacy', () => { + for (let index = 0; index < 12; index++) api.it(`spec ${index}`, body); + }); + + expect(() => selectDesktopSpecs(registry, environment, {})).toThrow( + /12 specs were not declared with desktopIt or desktopDescribe: "legacy spec 0", .*"legacy spec 9" and 2 more\./, + ); + expect(registry.blocked).toBe(true); + expect(runs('declared')).toBe(false); + }); + + test('a lane cannot be verified without Jasmine, so it blocks', () => { + const { registry } = setup('physical'); + + expect(() => selectDesktopSpecs(registry, undefined, {})).toThrow('Jasmine did not expose its environment'); + expect(registry.blocked).toBe(true); + }); + + test('without a lane, undeclared specs are tolerated', () => { + const { api, environment, registry } = setup(); + api.it('legacy', body); + + expect(() => selectDesktopSpecs(registry, environment, {})).not.toThrow(); + }); + + test('a fault found while registering blocks the run even without a lane, together with any other problem', () => { + const { environment, registry } = setup('accessible'); + registry.errors.push('Jasmine did not identify the spec registered for "ghost", so its selection cannot be verified.'); + + expect(() => selectDesktopSpecs(registry, environment, {})).toThrow(/ghost[\s\S]*accessible input lane/); + expect(registry.blocked).toBe(true); + }); +}); diff --git a/packages/agentic/desktop-test/src/jasmine/specSelection.ts b/packages/agentic/desktop-test/src/jasmine/specSelection.ts new file mode 100644 index 0000000000..ac3c059544 --- /dev/null +++ b/packages/agentic/desktop-test/src/jasmine/specSelection.ts @@ -0,0 +1,96 @@ +import { + missingDesktopRequirements, + resolveDesktopClickMode, + unmaskedAccessibleFeatures, +} from '@fluentui-react-native/desktop-driver/authoring'; +import { desktopSpecIdentifier } from './registry.js'; +import type { DesktopJasmineRegistry } from './registry.js'; + +type JasmineNode = { id?: unknown; description?: unknown; children?: readonly JasmineNode[] }; +type JasmineSpecFilter = (spec: unknown) => boolean; + +/** The part of Jasmine's environment that applying a selection needs. */ +export type DesktopJasmineEnvironment = { + configuration(): { specFilter?: JasmineSpecFilter }; + configure(configuration: { specFilter: JasmineSpecFilter }): void; + topSuite(): JasmineNode; +}; + +/** The environment of the Jasmine this process runs under, if a framework installed one. */ +export function currentDesktopJasmineEnvironment(): DesktopJasmineEnvironment | undefined { + const jasmine = (globalThis as { jasmine?: { getEnv?: () => DesktopJasmineEnvironment } }).jasmine; + return jasmine?.getEnv?.(); +} + +const listedSpecs = 10; + +/** + * Applies a run's selection once the spec files have registered their tests and the host is known, before any test runs. + * + * A fault that makes the selection untrustworthy disables every spec and throws, so a lane never executes a test it cannot + * vouch for. Otherwise, a test whose requirements the host cannot meet is excluded here and reported as unsupported. + */ +export function selectDesktopSpecs( + registry: DesktopJasmineRegistry, + environment: DesktopJasmineEnvironment | undefined, + features: unknown, +): void { + const lane = registry.selection?.lane; + const problems = [...registry.errors]; + if (lane === 'accessible') { + const unmasked = unmaskedAccessibleFeatures(features); + if (unmasked.length > 0) { + problems.push(`The accessible input lane needs a host that reports ${unmasked.join(', ')} disabled, and this one does not.`); + } + } + if (lane) { + const undeclared = environment ? undeclaredSpecs(registry, environment) : undefined; + if (!undeclared) problems.push('Jasmine did not expose its environment, so the specs of an input lane cannot be verified.'); + else if (undeclared.length > 0) { + const listed = undeclared.slice(0, listedSpecs).join(', '); + const more = undeclared.length > listedSpecs ? ` and ${undeclared.length - listedSpecs} more` : ''; + problems.push( + `An input lane runs only declared tests, but ${undeclared.length} ${undeclared.length === 1 ? 'spec was' : 'specs were'} not declared with desktopIt or desktopDescribe: ${listed}${more}.`, + ); + } + } + if (problems.length > 0) { + registry.blocked = true; + environment?.configure({ specFilter: () => false }); + throw new Error(problems.join('\n')); + } + if (!environment) return; + // The session's click mode is the lane's, so this is what a `click` requirement would really do. + const clickMode = resolveDesktopClickMode(undefined, 'auto', lane); + const unavailable = new Set(); + for (const [identifier, entry] of registry.entries) { + if (entry.kind !== 'run') continue; + const missing = missingDesktopRequirements(entry.descriptor, features as Readonly> | undefined, clickMode); + if (missing.length === 0) continue; + entry.missing = missing; + unavailable.add(identifier); + } + if (unavailable.size === 0) return; + const configured = environment.configuration().specFilter; + // A filtered spec is excluded before Jasmine runs its hooks, which an executed-then-skipped spec could not avoid. + environment.configure({ + specFilter: (spec) => { + const identifier = desktopSpecIdentifier(spec as { description?: unknown; id?: unknown }); + return !(identifier !== undefined && unavailable.has(identifier)) && (configured?.(spec) ?? true); + }, + }); +} + +/** The full names of the specs no `desktopIt` registered. */ +function undeclaredSpecs(registry: DesktopJasmineRegistry, environment: DesktopJasmineEnvironment): string[] { + const undeclared: string[] = []; + const visit = (node: JasmineNode, path: readonly string[]) => { + for (const child of node.children ?? []) { + const names = [...path, String(child.description)]; + if (child.children) visit(child, names); + else if (!registry.entries.has(desktopSpecIdentifier(child) ?? '')) undeclared.push(`"${names.join(' ')}"`); + } + }; + visit(environment.topSuite(), []); + return undeclared; +} diff --git a/packages/agentic/desktop-test/src/reporting/collector.test.ts b/packages/agentic/desktop-test/src/reporting/collector.test.ts index 40d556cf07..3d2d3f3f39 100644 --- a/packages/agentic/desktop-test/src/reporting/collector.test.ts +++ b/packages/agentic/desktop-test/src/reporting/collector.test.ts @@ -3,6 +3,127 @@ import os from 'node:os'; import path from 'node:path'; import { DesktopRunReporter } from './collector.js'; +import type { DesktopInputLaneReport } from './results.js'; + +describe('desktop run reporter input lanes', () => { + const accessible: DesktopInputLaneReport = { input: 'accessible', source: 'option', masking: 'host-masked', policyEnforced: false }; + let artifactsRoot: string; + + beforeEach(() => { + artifactsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-desktop-lane-')); + }); + + afterEach(() => { + fs.rmSync(artifactsRoot, { force: true, recursive: true }); + }); + + function start(reporterMode: 'plain' | 'ndjson', lane?: DesktopInputLaneReport) { + let written = ''; + const output = { + write: (chunk: string) => { + written += chunk; + return true; + }, + }; + const reporter = new DesktopRunReporter({ + artifactsRoot, + errorOutput: output, + output, + reporterMode, + summary: false, + ...(lane ? { lane } : {}), + }); + return { reporter, output: () => written }; + } + + function unsupported(reporter: DesktopRunReporter, testId: string, gap?: 'unsupported' | 'quarantine'): void { + reporter.emit({ + kind: 'test-result', + payload: { + testId, + title: testId, + attempt: 0, + status: 'unsupported', + durationMs: 0, + reason: 'Not available.', + ...(gap ? { gap } : {}), + }, + }); + } + + function passed(reporter: DesktopRunReporter, testId: string): void { + reporter.emit({ kind: 'test-result', payload: { testId, title: testId, attempt: 0, status: 'passed', durationMs: 1 } }); + } + + test('persists the lane and sums exclusions from every producer without selecting them as tests', async () => { + const { reporter } = start('plain', accessible); + reporter.emit({ kind: 'selection', payload: { excluded: { platform: 2, scope: 0, input: 5 } } }); + reporter.emit({ kind: 'selection', payload: { excluded: { platform: 1, scope: 1, input: 4 } } }); + passed(reporter, 'case'); + const result = await reporter.finish(); + + expect(result).toMatchObject({ + status: 'passed', + lane: accessible, + selection: { excluded: { platform: 3, scope: 1, input: 9 } }, + counts: { selected: 1, passed: 1 }, + }); + expect(JSON.parse(fs.readFileSync(path.join(reporter.root, 'run.json'), 'utf8'))).toMatchObject({ + lane: accessible, + selection: { excluded: { platform: 3, scope: 1, input: 9 } }, + }); + }); + + test('counts a keyed selection once, so a retried worker does not double its exclusions', async () => { + const { reporter } = start('plain', accessible); + const excluded = { platform: 1, scope: 2, input: 3 }; + reporter.emit({ kind: 'selection', payload: { excluded, key: 'wdio-group' } }); + reporter.emit({ kind: 'selection', payload: { excluded, key: 'wdio-group' } }); + reporter.emit({ kind: 'selection', payload: { excluded: { platform: 0, scope: 0, input: 1 }, key: 'another-group' } }); + passed(reporter, 'case'); + + expect(await reporter.finish()).toMatchObject({ selection: { excluded: { platform: 1, scope: 2, input: 4 } } }); + }); + + test('omits the selection when no producer reported one', async () => { + const { reporter } = start('plain', accessible); + passed(reporter, 'case'); + + expect(await reporter.finish()).not.toHaveProperty('selection'); + }); + + test('a lane run is incomplete when a case was unsupported without a declaration', async () => { + const { reporter, output } = start('plain', accessible); + passed(reporter, 'case'); + unsupported(reporter, 'declared', 'unsupported'); + unsupported(reporter, 'missing'); + reporter.emit({ kind: 'selection', payload: { excluded: { platform: 0, scope: 0, input: 3 } } }); + const result = await reporter.finish(); + + expect(result.status).toBe('incomplete'); + expect(output()).toContain('2 unsupported (1 declared, 0 quarantined)'); + expect(output()).toContain('[desktop] input accessible (host-masked) from --input; excluded 0 by platform, 0 by scope, 3 by input'); + }); + + test('the same unsupported case does not fail a compatibility run', async () => { + const { reporter, output } = start('plain'); + passed(reporter, 'case'); + unsupported(reporter, 'missing'); + const result = await reporter.finish(); + + expect(result.status).toBe('passed'); + expect(result).not.toHaveProperty('lane'); + expect(output()).not.toContain('excluded'); + }); + + test('the ndjson start message names the lane', async () => { + const { reporter, output } = start('ndjson', accessible); + passed(reporter, 'case'); + await reporter.finish(); + + expect(JSON.parse(output().split('\n')[0])).toMatchObject({ type: 'start', lane: accessible }); + }); +}); describe('desktop run reporter final output', () => { let artifactsRoot: string; diff --git a/packages/agentic/desktop-test/src/reporting/collector.ts b/packages/agentic/desktop-test/src/reporting/collector.ts index 4acbd00b8d..c2d81db4cf 100644 --- a/packages/agentic/desktop-test/src/reporting/collector.ts +++ b/packages/agentic/desktop-test/src/reporting/collector.ts @@ -9,12 +9,20 @@ import { DesktopReportedError, redactDesktopSecrets, sanitizeDesktopReport, + selectionExclusionReasons, serializeReportError, validateDesktopEvent, } from './protocol.js'; import type { CollectedDesktopEvent, EventContext, EventEmitter, FailureKind, ReportError, ReportFailure } from './protocol.js'; -import { projectDesktopRunSummary, renderJUnit, renderRunSummary, summarizeDesktopRun } from './results.js'; -import type { DesktopRunResult, DesktopTestResult, DesktopWorkResult } from './results.js'; +import { + describeDesktopSelection, + describeDesktopUnsupported, + projectDesktopRunSummary, + renderJUnit, + renderRunSummary, + summarizeDesktopRun, +} from './results.js'; +import type { DesktopInputLaneReport, DesktopRunResult, DesktopTestResult, DesktopWorkResult } from './results.js'; import { includeDesktopEvent, projectDesktopEvent } from './structured.js'; import type { DesktopEventLevel } from './structured.js'; import type { DesktopReportingConfig } from '../config.js'; @@ -25,6 +33,8 @@ export type ReportOutput = Pick; export type DesktopRunReporterOptions = DesktopReportingConfig & { artifactsRoot: string; context?: EventContext; + /** The input lane this run executes. Lane runs hold every selected case to its declared outcome. */ + lane?: DesktopInputLaneReport; kind?: 'tests' | 'lifecycle'; optional?: boolean; output?: ReportOutput; @@ -68,6 +78,9 @@ export class DesktopRunReporter { private readonly presentedFailures = new Set(); private readonly secrets = new Set(); private readonly artifacts = new Map(); + private readonly excluded = { platform: 0, scope: 0, input: 0 }; + private readonly selectionKeys = new Set(); + private excludedReported = false; private readonly options: DesktopRunReporterOptions; private readonly actions: boolean; private readonly output: ReportOutput; @@ -136,6 +149,7 @@ export class DesktopRunReporter { runId: this.runId, kind: options.kind ?? 'tests', ...(options.context ? { context: options.context } : {}), + ...(options.lane ? { lane: options.lane } : {}), artifactsRoot: clipStructuredText(this.root, 512), }; this.enqueue(() => this.write(this.output, `${JSON.stringify(start)}\n`), 0); @@ -314,6 +328,14 @@ export class DesktopRunReporter { test.status = test.attempts[test.attempts.length - 1].status; test.flaky = test.status === 'passed' && test.attempts.some(({ status }) => status === 'failed'); this.tests.set(p.testId, test); + } else if (event.kind === 'selection') { + this.excludedReported = true; + // A keyed event is idempotent: a retried worker announces the same exclusions again. + if (event.payload.key !== undefined) { + if (this.selectionKeys.has(event.payload.key)) return; + this.selectionKeys.add(event.payload.key); + } + for (const reason of selectionExclusionReasons) this.excluded[reason] += event.payload.excluded[reason]; } else if (event.kind === 'artifact') this.artifacts.set(event.payload.path, event.payload.description); } @@ -495,6 +517,8 @@ export class DesktopRunReporter { finishedAt: new Date().toISOString(), durationMs: Date.now() - this.startedAt.getTime(), context: this.options.context, + ...(this.options.lane ? { lane: this.options.lane } : {}), + ...(this.excludedReported ? { selection: { excluded: { ...this.excluded } } } : {}), optional: this.options.optional, incomplete: [...this.scopes.values()].some(({ finished }) => !finished) || @@ -551,8 +575,10 @@ export class DesktopRunReporter { let result = persist(); const c = result.counts; this.logger.log( - `[desktop] ${result.status.toUpperCase()}: ${c.selected} selected, ${c.executed} executed, ${c.passed} passed, ${c.failed} failed, ${c.skipped} skipped, ${c.unsupported} unsupported, ${c.incomplete} incomplete, ${c.retries} retries, ${c.flaky} flaky; ${result.work.filter(({ status }) => status === 'not-run').length} groups not run; ${path.join(this.root, 'run.json')}`, + `[desktop] ${result.status.toUpperCase()}: ${c.selected} selected, ${c.executed} executed, ${c.passed} passed, ${c.failed} failed, ${c.skipped} skipped, ${describeDesktopUnsupported(c)}, ${c.incomplete} incomplete, ${c.retries} retries, ${c.flaky} flaky; ${result.work.filter(({ status }) => status === 'not-run').length} groups not run; ${path.join(this.root, 'run.json')}`, ); + const selection = describeDesktopSelection(result); + if (selection) this.logger.log(`[desktop] ${selection}`); await this.queue; result = persist(); if (this.presentationFailed && !this.structured) { diff --git a/packages/agentic/desktop-test/src/reporting/index.ts b/packages/agentic/desktop-test/src/reporting/index.ts index 428250bac3..16383d62c2 100644 --- a/packages/agentic/desktop-test/src/reporting/index.ts +++ b/packages/agentic/desktop-test/src/reporting/index.ts @@ -11,11 +11,33 @@ export type { FailureKind, ReportError, ReportFailure, + SelectionExclusions, + TestGap, TestStatus, } from './protocol.js'; export type { DesktopEventRecord } from './structured.js'; export type { DesktopEventLevel } from '@fluentui-react-native/desktop-driver/cli/reporter'; -export { desktopRunExitCode, projectDesktopRunSummary, renderJUnit, renderRunSummary, summarizeDesktopRun } from './results.js'; -export type { DesktopRunResult, DesktopRunSummary, DesktopTestAttempt, DesktopTestResult } from './results.js'; +export { createDesktopInputLane, desktopInputEnvironment, parseDesktopInputLane, requestedDesktopInputLane } from './lane.js'; +export type { DesktopInputLaneName, RequestedDesktopInputLane } from './lane.js'; +export { + describeDesktopInputLane, + describeDesktopSelection, + describeDesktopUnsupported, + desktopInputLaneSources, + desktopRunExitCode, + projectDesktopRunSummary, + renderJUnit, + renderRunSummary, + summarizeDesktopRun, +} from './results.js'; +export type { + DesktopInputLaneReport, + DesktopInputLaneSource, + DesktopRunResult, + DesktopRunSummary, + DesktopSelectionReport, + DesktopTestAttempt, + DesktopTestResult, +} from './results.js'; export { connectReportProducer, DESKTOP_REPORT_CONTEXT, startReportTransport } from './transport.js'; export type { ReportProducer, ReportTransport, ReportTransportContext } from './transport.js'; diff --git a/packages/agentic/desktop-test/src/reporting/lane.test.ts b/packages/agentic/desktop-test/src/reporting/lane.test.ts new file mode 100644 index 0000000000..f10c320dd9 --- /dev/null +++ b/packages/agentic/desktop-test/src/reporting/lane.test.ts @@ -0,0 +1,34 @@ +import { createDesktopInputLane, desktopInputEnvironment, parseDesktopInputLane, requestedDesktopInputLane } from './lane.js'; + +describe('requested desktop input lane', () => { + test('the flag wins over the environment', () => { + expect(requestedDesktopInputLane({ flag: 'physical', environment: 'accessible' })).toEqual({ input: 'physical', source: 'option' }); + }); + + test('the environment applies when there is no flag', () => { + expect(requestedDesktopInputLane({ environment: 'accessible' })).toEqual({ input: 'accessible', source: 'environment' }); + expect(requestedDesktopInputLane({})).toBeUndefined(); + }); + + test('an empty value is invalid rather than unset, and an error names where it came from', () => { + expect(() => requestedDesktopInputLane({ environment: '' })).toThrow(`${desktopInputEnvironment} must be accessible or physical.`); + expect(() => requestedDesktopInputLane({ flag: 'fast' })).toThrow('--input must be accessible or physical.'); + expect(() => parseDesktopInputLane('Physical', 'x')).toThrow(expect.objectContaining({ code: 'config' })); + }); +}); + +describe('createDesktopInputLane', () => { + test('an accessible lane says how it was masked, and neither lane claims enforcement yet', () => { + expect(createDesktopInputLane('accessible', 'option')).toEqual({ + input: 'accessible', + source: 'option', + masking: 'host-masked', + policyEnforced: false, + }); + expect(createDesktopInputLane('accessible', 'driver', 'verified-masked')).toMatchObject({ + masking: 'verified-masked', + source: 'driver', + }); + expect(createDesktopInputLane('physical', 'environment')).toEqual({ input: 'physical', source: 'environment', policyEnforced: false }); + }); +}); diff --git a/packages/agentic/desktop-test/src/reporting/lane.ts b/packages/agentic/desktop-test/src/reporting/lane.ts new file mode 100644 index 0000000000..49064f01e2 --- /dev/null +++ b/packages/agentic/desktop-test/src/reporting/lane.ts @@ -0,0 +1,37 @@ +import type { DesktopInputLaneReport, DesktopInputLaneSource } from './results.js'; + +/** Selects the input lane of a `test` run when `--input` is absent. */ +export const desktopInputEnvironment = 'DESKTOP_INPUT'; + +export type DesktopInputLaneName = DesktopInputLaneReport['input']; + +export type RequestedDesktopInputLane = { input: DesktopInputLaneName; source: Extract }; + +/** Validates a requested lane. `source` names where the value came from, so a bad one points at the right place. */ +export function parseDesktopInputLane(value: string, source: string): DesktopInputLaneName { + if (value !== 'accessible' && value !== 'physical') { + throw Object.assign(new TypeError(`${source} must be accessible or physical.`), { code: 'config' }); + } + return value; +} + +/** The lane a command was asked for: `--input`, then `DESKTOP_INPUT`. An empty environment value is invalid, not unset. */ +export function requestedDesktopInputLane(request: { flag?: string; environment?: string }): RequestedDesktopInputLane | undefined { + if (request.flag !== undefined) return { input: parseDesktopInputLane(request.flag, '--input'), source: 'option' }; + if (request.environment !== undefined) { + return { input: parseDesktopInputLane(request.environment, desktopInputEnvironment), source: 'environment' }; + } + return undefined; +} + +/** + * How an accessible lane was masked. `host-masked` means this command started the host with keyboard, physical click and + * wheel disabled. `verified-masked` means it attached to a host and checked the mask before running anything. + */ +export function createDesktopInputLane( + input: DesktopInputLaneName, + source: DesktopInputLaneSource, + masking: 'host-masked' | 'verified-masked' = 'host-masked', +): DesktopInputLaneReport { + return input === 'physical' ? { input, source, policyEnforced: false } : { input, source, masking, policyEnforced: false }; +} diff --git a/packages/agentic/desktop-test/src/reporting/protocol.test.ts b/packages/agentic/desktop-test/src/reporting/protocol.test.ts index 19f5fc1fac..2462042655 100644 --- a/packages/agentic/desktop-test/src/reporting/protocol.test.ts +++ b/packages/agentic/desktop-test/src/reporting/protocol.test.ts @@ -1,4 +1,52 @@ -import { redactDesktopSecrets } from './protocol'; +import { redactDesktopSecrets, validateDesktopEvent } from './protocol'; + +describe('declared gaps and selection events', () => { + function event(kind: string, payload: Record): unknown { + return { + schemaVersion: 1, + runId: 'run-id', + eventId: 'event-id', + sourceId: 'worker', + sourceSequence: 1, + scopeId: 'scope-id', + timestamp: new Date(0).toISOString(), + kind, + payload, + }; + } + const result = { testId: 'test', title: 'test', attempt: 0, durationMs: 0, reason: 'Declared in the story.' }; + + test.each(['unsupported', 'quarantine'])('accepts a %s gap on an unsupported result', (gap) => { + expect(() => validateDesktopEvent(event('test-result', { ...result, status: 'unsupported', gap }))).not.toThrow(); + }); + + test.each(['skipped', 'failed', 'passed'])('rejects a declared gap on a %s result', (status) => { + expect(() => validateDesktopEvent(event('test-result', { ...result, status, gap: 'unsupported' }))).toThrow(); + }); + + test('rejects an unknown gap kind', () => { + expect(() => validateDesktopEvent(event('test-result', { ...result, status: 'unsupported', gap: 'flaky' }))).toThrow(); + }); + + test('accepts exactly the three non-negative integer exclusion counts', () => { + expect(() => validateDesktopEvent(event('selection', { excluded: { platform: 1, scope: 0, input: 12 } }))).not.toThrow(); + }); + + test('accepts an optional idempotency key and rejects an empty one', () => { + const excluded = { platform: 1, scope: 0, input: 0 }; + expect(() => validateDesktopEvent(event('selection', { excluded, key: 'wdio-group' }))).not.toThrow(); + expect(() => validateDesktopEvent(event('selection', { excluded, key: '' }))).toThrow(); + }); + + test.each([ + ['a missing reason', { platform: 1, scope: 0 }], + ['an unknown reason', { platform: 1, scope: 0, input: 0, flaky: 1 }], + ['a negative count', { platform: -1, scope: 0, input: 0 }], + ['a fractional count', { platform: 0.5, scope: 0, input: 0 }], + ])('rejects a selection with %s', (_name, excluded) => { + expect(() => validateDesktopEvent(event('selection', { excluded }))).toThrow(); + }); +}); describe('redactDesktopSecrets', () => { test('redacts known secrets, longest first', () => { diff --git a/packages/agentic/desktop-test/src/reporting/protocol.ts b/packages/agentic/desktop-test/src/reporting/protocol.ts index 447455b715..524c10f50b 100644 --- a/packages/agentic/desktop-test/src/reporting/protocol.ts +++ b/packages/agentic/desktop-test/src/reporting/protocol.ts @@ -15,6 +15,12 @@ export class DesktopReportedError extends Error { } } export type TestStatus = 'passed' | 'failed' | 'skipped' | 'unsupported' | 'cancelled' | 'not-run' | 'incomplete'; +/** A declared gap: the producer knows the case cannot run here and says why, so it is not a silent skip. */ +export type TestGap = 'unsupported' | 'quarantine'; +export const testGaps: readonly TestGap[] = ['unsupported', 'quarantine']; +/** Cases a producer removed from its selection before any session existed, by the reason they were removed. */ +export type SelectionExclusions = { platform: number; scope: number; input: number }; +export const selectionExclusionReasons: readonly (keyof SelectionExclusions)[] = ['platform', 'scope', 'input']; export type EventContext = { endpoint?: 'macos' | 'windows' | 'win32'; renderer?: 'fabric' | 'paper'; @@ -43,9 +49,11 @@ export type EventData = status: TestStatus; durationMs: number; reason?: string; + gap?: TestGap; failure?: ReportFailure; }; } + | { kind: 'selection'; payload: { excluded: SelectionExclusions; key?: string } } | { kind: 'artifact'; payload: { path: string; description: string } } | { kind: 'producer-finished'; payload: { status: 'passed' | 'failed' } } | { kind: 'run-finished'; payload: { status: TestStatus; resultPath: string } }; @@ -74,7 +82,8 @@ const payloadFields = { diagnostic: ['level', 'message', 'failure'], output: ['stream', 'text'], 'test-start': ['testId', 'title', 'attempt'], - 'test-result': ['testId', 'title', 'attempt', 'status', 'durationMs', 'reason', 'failure'], + 'test-result': ['testId', 'title', 'attempt', 'status', 'durationMs', 'reason', 'gap', 'failure'], + selection: ['excluded', 'key'], artifact: ['path', 'description'], 'producer-finished': ['status'], 'run-finished': ['status', 'resultPath'], @@ -178,9 +187,20 @@ export function validateDesktopEvent(value: unknown): asserts value is DesktopEv throw new TypeError('Non-executed test results require a reason.'); } if (payload.status === 'failed' && !payload.failure) throw new TypeError('Failed test results require their original failure.'); + if (payload.gap !== undefined) { + member(payload.gap, testGaps); + if (payload.status !== 'unsupported') throw new TypeError('A declared gap is only valid on an unsupported test result.'); + } if (payload.failure !== undefined) validateFailure(payload.failure); } break; + case 'selection': { + const excluded = record(payload.excluded, 'excluded'); + exactKeys(excluded, selectionExclusionReasons); + for (const reason of selectionExclusionReasons) integer(excluded[reason], `excluded.${reason}`, 0); + if (payload.key !== undefined) requireText(payload.key, 'selection key'); + break; + } case 'artifact': requireText(payload.path, 'artifact path'); requireText(payload.description, 'artifact description'); diff --git a/packages/agentic/desktop-test/src/reporting/results.test.ts b/packages/agentic/desktop-test/src/reporting/results.test.ts index af29cd80c3..82208acd89 100644 --- a/packages/agentic/desktop-test/src/reporting/results.test.ts +++ b/packages/agentic/desktop-test/src/reporting/results.test.ts @@ -1,7 +1,174 @@ import os from 'node:os'; import path from 'node:path'; -import { projectDesktopRunSummary, summarizeDesktopRun } from './results.js'; +import { + describeDesktopSelection, + describeDesktopUnsupported, + projectDesktopRunSummary, + renderJUnit, + renderRunSummary, + summarizeDesktopRun, +} from './results.js'; +import type { DesktopInputLaneReport, DesktopRunResult, DesktopTestResult } from './results.js'; + +const accessible: DesktopInputLaneReport = { input: 'accessible', source: 'option', masking: 'host-masked', policyEnforced: false }; + +function outcome( + status: DesktopTestResult['status'], + gap?: 'unsupported' | 'quarantine', + testId = `${status}-${gap ?? 'plain'}`, +): DesktopTestResult { + return { + testId, + title: testId, + status, + flaky: false, + attempts: [ + { + attempt: 0, + status, + durationMs: 1, + ...(status === 'passed' ? {} : { reason: 'Not executed.' }), + ...(gap ? { gap } : {}), + }, + ], + }; +} + +function summarize( + tests: DesktopTestResult[], + extra: { lane?: DesktopInputLaneReport; optional?: boolean; kind?: 'tests' | 'lifecycle' } = {}, +): DesktopRunResult { + const { lane, optional, kind = 'tests' } = extra; + return summarizeDesktopRun({ + schemaVersion: 1, + runId: 'run-id', + kind, + startedAt: new Date(0).toISOString(), + finishedAt: new Date(1).toISOString(), + durationMs: 1, + ...(lane ? { lane } : {}), + ...(optional !== undefined ? { optional } : {}), + tests, + work: [], + failures: [], + artifacts: [], + }); +} + +describe('input lane verdicts', () => { + test('compatibility runs keep skipped and unsupported cases non-blocking', () => { + expect(summarize([outcome('passed'), outcome('skipped'), outcome('unsupported')]).status).toBe('passed'); + }); + + test.each([ + ['unsupported', 'declaredUnsupported'], + ['quarantine', 'quarantined'], + ] as const)('a declared %s gap is a deliberate gap, not a lane failure', (gap, subset) => { + const result = summarize([outcome('passed'), outcome('unsupported', gap)], { lane: accessible }); + + expect(result.status).toBe('passed'); + expect(result.counts).toMatchObject({ + unsupported: 1, + declaredUnsupported: Number(gap === 'unsupported'), + quarantined: Number(gap === 'quarantine'), + }); + expect(result.counts[subset]).toBe(1); + }); + + test('an undeclared unsupported case means the lane did not cover its selection', () => { + const result = summarize([outcome('passed'), outcome('unsupported')], { lane: accessible }); + + expect(result.status).toBe('incomplete'); + expect(result.counts).toMatchObject({ unsupported: 1, declaredUnsupported: 0, quarantined: 0 }); + }); + + test('a skipped case means the lane did not cover its selection', () => { + expect(summarize([outcome('passed'), outcome('skipped')], { lane: accessible }).status).toBe('incomplete'); + }); + + test('optional never masks a lane that skipped or executed nothing', () => { + expect(summarize([outcome('passed'), outcome('skipped')], { lane: accessible, optional: true }).status).toBe('incomplete'); + expect(summarize([outcome('unsupported', 'unsupported')], { lane: accessible, optional: true }).status).toBe('incomplete'); + expect(summarize([outcome('skipped')], { optional: true }).status).toBe('skipped'); + }); + + test('an empty lane selection is incomplete', () => { + expect(summarize([], { lane: accessible }).status).toBe('incomplete'); + }); + + test('failures still fail a lane run', () => { + const failed = outcome('failed'); + failed.attempts[0].failure = { failureId: 'f', kind: 'test', error: { name: 'Error', message: 'boom' } }; + + expect(summarize([outcome('passed'), failed], { lane: accessible }).status).toBe('failed'); + }); + + test('lifecycle runs ignore the lane', () => { + expect(summarize([], { lane: accessible, kind: 'lifecycle' }).status).toBe('passed'); + }); +}); + +describe('input lane and exclusion reporting', () => { + const selection = { excluded: { platform: 2, scope: 1, input: 7 } }; + const result = (): DesktopRunResult => ({ + ...summarize([outcome('passed'), outcome('unsupported', 'unsupported'), outcome('unsupported', 'quarantine', 'quarantined-case')], { + lane: accessible, + }), + selection, + }); + + test('names the lane, its source and each exclusion on one line', () => { + expect(describeDesktopSelection(result())).toBe( + 'input accessible (host-masked) from --input; excluded 2 by platform, 1 by scope, 7 by input', + ); + expect(describeDesktopSelection({ lane: { input: 'physical', source: 'environment', policyEnforced: false } })).toBe( + 'input physical pilot from DESKTOP_INPUT', + ); + expect( + describeDesktopSelection({ lane: { input: 'accessible', source: 'driver', masking: 'verified-masked', policyEnforced: false } }), + ).toBe('input accessible (verified-masked) from the driver instance'); + expect(describeDesktopSelection({})).toBeUndefined(); + }); + + test('breaks unsupported cases into declared and quarantined', () => { + expect(describeDesktopUnsupported(result().counts)).toBe('2 unsupported (1 declared, 1 quarantined)'); + }); + + test('JUnit records the lane and exclusions and labels declared gaps', () => { + const xml = renderJUnit(result()); + + expect(xml).toContain(''); + expect(xml).toContain(''); + expect(xml).toContain(''); + expect(xml).toContain(''); + expect(xml).toContain(''); + expect(xml).toContain(''); + expect(xml).toContain(''); + expect(xml).toContain('skipped="2"'); + }); + + test('JUnit stays free of lane properties for compatibility runs', () => { + expect(renderJUnit(summarize([outcome('passed')]))).not.toContain(''); + }); + + test('the Markdown summary leads with the selection and splits unsupported counts', () => { + const markdown = renderRunSummary(result()); + + expect(markdown).toContain('Selection: input accessible (host-masked) from --input; excluded 2 by platform, 1 by scope, 7 by input.'); + expect(markdown).toContain('2 unsupported (1 declared, 1 quarantined)'); + expect(markdown).toContain('declared unsupported: Not executed.'); + }); + + test('the bounded summary carries the lane and exclusions', () => { + const summary = projectDesktopRunSummary(result(), path.join(os.tmpdir(), 'desktop-artifacts', 'run-id')); + + expect(summary.lane).toEqual(accessible); + expect(summary.excluded).toEqual(selection.excluded); + expect(summary.skippedReasons).toContain('declared unsupported: Not executed.'); + expect(Buffer.byteLength(JSON.stringify(summary))).toBeLessThanOrEqual(4 * 1024); + }); +}); describe('bounded desktop run summary', () => { test('keeps large suites and failure messages within the document limit', () => { diff --git a/packages/agentic/desktop-test/src/reporting/results.ts b/packages/agentic/desktop-test/src/reporting/results.ts index 2470c157bc..02fa673fe8 100644 --- a/packages/agentic/desktop-test/src/reporting/results.ts +++ b/packages/agentic/desktop-test/src/reporting/results.ts @@ -1,15 +1,28 @@ import path from 'node:path'; -import type { EventContext, ReportFailure, TestStatus } from './protocol.js'; +import type { EventContext, ReportFailure, SelectionExclusions, TestGap, TestStatus } from './protocol.js'; export type DesktopTestAttempt = { attempt: number; status: TestStatus; durationMs: number; reason?: string; + /** Present only on an `unsupported` attempt the producer declared in advance, never on a missing requirement. */ + gap?: TestGap; failure?: ReportFailure; context?: EventContext; }; +/** + * The input lane a run was asked to execute. It records the request and how the host was masked; it never claims what + * input a session delivered or enforced, hence the literal `policyEnforced: false`. + */ +export const desktopInputLaneSources = ['option', 'environment', 'driver'] as const; +/** `driver` means the lane was read from the attached driver instance's manifest instead of being requested here. */ +export type DesktopInputLaneSource = (typeof desktopInputLaneSources)[number]; +export type DesktopInputLaneReport = + | { input: 'accessible'; source: DesktopInputLaneSource; masking: 'host-masked' | 'verified-masked'; policyEnforced: false } + | { input: 'physical'; source: DesktopInputLaneSource; policyEnforced: false }; +export type DesktopSelectionReport = { excluded: SelectionExclusions }; export type DesktopTestResult = { testId: string; title: string; @@ -34,13 +47,22 @@ export type DesktopRunResult = { finishedAt: string; durationMs: number; context?: EventContext; + /** Set when the run executes one input lane. Lane runs treat undeclared skips and unsupported cases as non-success. */ + lane?: DesktopInputLaneReport; + /** What producers removed from their selection before execution. Excluded cases are not tests. */ + selection?: DesktopSelectionReport; counts: { selected: number; executed: number; passed: number; failed: number; skipped: number; + /** Every test whose final status is `unsupported`, declared or not. */ unsupported: number; + /** The subset of `unsupported` that carries a declared `unsupported` gap. */ + declaredUnsupported: number; + /** The subset of `unsupported` that carries a declared `quarantine` gap. */ + quarantined: number; incomplete: number; retries: number; flaky: number; @@ -59,6 +81,8 @@ export type DesktopRunSummary = { status: TestStatus; durationMs: number; context?: Pick; + lane?: DesktopInputLaneReport; + excluded?: SelectionExclusions; counts: DesktopRunResult['counts']; problems: { kind: 'test' | 'infrastructure' | 'unfinished'; @@ -122,7 +146,7 @@ export function projectDesktopRunSummary(result: DesktopRunResult, root: string) ...new Set( result.tests.flatMap((test) => test.attempts.flatMap((attempt) => - attempt.reason && ['skipped', 'unsupported'].includes(attempt.status) ? [clipText(attempt.reason, 256)] : [], + attempt.reason && ['skipped', 'unsupported'].includes(attempt.status) ? [clipText(describeReason(attempt), 256)] : [], ), ), ), @@ -152,6 +176,8 @@ export function projectDesktopRunSummary(result: DesktopRunResult, root: string) } : {}), counts: result.counts, + ...(result.lane ? { lane: result.lane } : {}), + ...(result.selection ? { excluded: result.selection.excluded } : {}), problems: problems.slice(0, SUMMARY_MAX_PROBLEMS), skippedReasons: skippedReasons.slice(0, SUMMARY_MAX_PROBLEMS), artifacts, @@ -205,6 +231,8 @@ export function summarizeDesktopRun( failed: 0, skipped: 0, unsupported: 0, + declaredUnsupported: 0, + quarantined: 0, incomplete: 0, retries: 0, flaky: 0, @@ -217,18 +245,26 @@ export function summarizeDesktopRun( else if (test.status === 'passed') counts.passed++; else if (test.status === 'failed') counts.failed++; else if (test.status === 'skipped') counts.skipped++; - else if (test.status === 'unsupported') counts.unsupported++; + else if (test.status === 'unsupported') { + counts.unsupported++; + const gap = test.attempts.at(-1)?.gap; + if (gap === 'unsupported') counts.declaredUnsupported++; + else if (gap === 'quarantine') counts.quarantined++; + } } const infrastructureFailed = input.failures.some(({ kind }) => kind !== 'test') || input.work.some(({ status }) => status === 'failed'); const unfinishedWork = input.work.some(({ status }) => ['not-run', 'incomplete', 'cancelled'].includes(status)); const noExecution = input.kind === 'tests' && counts.executed === 0; + // A lane run promised to execute its whole selection. Only a declared gap may leave a selected case unexecuted, and a + // run that executed nothing is never optional. + const laneUnmet = input.kind === 'tests' && input.lane !== undefined && counts.skipped + undeclaredUnsupported(counts) > 0; const status = infrastructureFailed || counts.failed > 0 ? 'failed' - : input.incomplete || unfinishedWork || counts.incomplete > 0 || (input.kind === 'tests' && counts.selected === 0) + : input.incomplete || unfinishedWork || counts.incomplete > 0 || laneUnmet || (input.kind === 'tests' && counts.selected === 0) ? 'incomplete' : noExecution - ? input.optional + ? input.optional && input.lane === undefined ? 'skipped' : 'incomplete' : 'passed'; @@ -236,6 +272,46 @@ export function summarizeDesktopRun( return { ...result, status, counts }; } +/** Unsupported cases nothing declared in advance: a requirement the target did not meet. */ +function undeclaredUnsupported(counts: Pick): number { + return counts.unsupported - counts.declaredUnsupported - counts.quarantined; +} + +const gapLabels = { unsupported: 'declared unsupported', quarantine: 'quarantined' } as const; + +/** A reason with its declared-gap label, so a declared gap never reads like a missing requirement. */ +function describeReason(attempt: Pick): string { + const reason = attempt.reason ?? ''; + return attempt.gap ? `${gapLabels[attempt.gap]}: ${reason}` : reason; +} + +/** `accessible (host-masked)`, `accessible (verified-masked)` or `physical pilot`; never a claim of enforced input. */ +export function describeDesktopInputLane(lane: DesktopInputLaneReport): string { + return lane.input === 'physical' ? 'physical pilot' : `accessible (${lane.masking})`; +} + +function describeLaneSource(lane: DesktopInputLaneReport): string { + return lane.source === 'option' ? '--input' : lane.source === 'environment' ? 'DESKTOP_INPUT' : 'the driver instance'; +} + +/** One line naming the lane and what each producer excluded before execution, or `undefined` when neither applies. */ +export function describeDesktopSelection(result: Pick): string | undefined { + const parts: string[] = []; + if (result.lane) parts.push(`input ${describeDesktopInputLane(result.lane)} from ${describeLaneSource(result.lane)}`); + if (result.selection) { + const { platform, scope, input } = result.selection.excluded; + parts.push(`excluded ${platform} by platform, ${scope} by scope, ${input} by input`); + } + return parts.length ? parts.join('; ') : undefined; +} + +/** The unsupported count with its declared and quarantined subsets, so undeclared gaps stand out. */ +export function describeDesktopUnsupported( + counts: Pick, +): string { + return `${counts.unsupported} unsupported (${counts.declaredUnsupported} declared, ${counts.quarantined} quarantined)`; +} + export function desktopRunExitCode(result: DesktopRunResult): number { return result.status === 'passed' || result.status === 'skipped' ? 0 : 1; } @@ -253,7 +329,7 @@ export function renderJUnit(result: DesktopRunResult): string { : test.status === 'passed' ? '' : ['skipped', 'unsupported'].includes(test.status) - ? `` + ? `` : ``; return `${body}${xml(history)}`; }); @@ -270,18 +346,34 @@ export function renderJUnit(result: DesktopRunResult): string { cases.push( '', ); - return `\n${cases.join('')}\n`; + const properties: [string, string][] = []; + if (result.lane) + properties.push( + ['desktop.input', result.lane.input], + ['desktop.input.label', describeDesktopInputLane(result.lane)], + ['desktop.input.source', result.lane.source], + ); + if (result.selection) + for (const [reason, count] of Object.entries(result.selection.excluded)) properties.push([`desktop.excluded.${reason}`, String(count)]); + if (result.counts.declaredUnsupported) properties.push(['desktop.declaredUnsupported', String(result.counts.declaredUnsupported)]); + if (result.counts.quarantined) properties.push(['desktop.quarantined', String(result.counts.quarantined)]); + const propertyXml = properties.length + ? `${properties.map(([name, value]) => ``).join('')}` + : ''; + return `\n${propertyXml}${cases.join('')}\n`; } export function renderRunSummary(result: DesktopRunResult): string { const c = result.counts; const reasons = [ - ...new Set(result.tests.flatMap((test) => test.attempts.flatMap((attempt) => (attempt.reason ? [attempt.reason] : [])))), + ...new Set(result.tests.flatMap((test) => test.attempts.flatMap((attempt) => (attempt.reason ? [describeReason(attempt)] : [])))), ]; + const selection = describeDesktopSelection(result); return ( `## Desktop tests: ${result.status}\n\n` + `Run: \`${result.runId}\`; endpoint: ${result.context?.endpoint ?? 'native'}; renderer: ${result.context?.renderer ?? 'not applicable'}.\n\n` + - `Selected ${c.selected}; executed ${c.executed}; passed ${c.passed}; failed ${c.failed}; skipped ${c.skipped}; unsupported ${c.unsupported}; incomplete ${c.incomplete}; retries ${c.retries}; flaky ${c.flaky}.\n\n` + + (selection ? `Selection: ${selection}.\n\n` : '') + + `Selected ${c.selected}; executed ${c.executed}; passed ${c.passed}; failed ${c.failed}; skipped ${c.skipped}; ${describeDesktopUnsupported(c)}; incomplete ${c.incomplete}; retries ${c.retries}; flaky ${c.flaky}.\n\n` + `Selected groups ${result.work.length}; not run ${result.work.filter(({ status }) => status === 'not-run').length}; incomplete ${result.work.filter(({ status }) => status === 'incomplete').length}.\n\n` + `Infrastructure/evidence/cleanup failures: ${result.failures.filter(({ kind }) => kind !== 'test').length}.\n\n` + (reasons.length diff --git a/packages/agentic/desktop-test/src/reporting/runQueries.test.ts b/packages/agentic/desktop-test/src/reporting/runQueries.test.ts index 75a352a566..b04dad2753 100644 --- a/packages/agentic/desktop-test/src/reporting/runQueries.test.ts +++ b/packages/agentic/desktop-test/src/reporting/runQueries.test.ts @@ -332,6 +332,95 @@ describe('persisted desktop run queries', () => { await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toThrow(); }); + describe('input lane runs', () => { + const lane = { input: 'accessible', source: 'environment', masking: 'verified-masked', policyEnforced: false } as const; + let laneReporter: DesktopRunReporter; + let laneClient: DesktopRunQueryClient; + + beforeEach(async () => { + laneReporter = new DesktopRunReporter({ + artifactsRoot: root, + output: { write: () => true }, + reporterMode: 'json', + summary: false, + lane, + }); + service.registerRun({ runId: laneReporter.runId, root: laneReporter.root, expiresAt: Date.now() + 60_000 }); + laneClient = service.connect(service.issueGrant({ runIds: [laneReporter.runId], expiresAt: Date.now() + 60_000 })); + laneReporter.emit({ kind: 'selection', payload: { excluded: { platform: 1, scope: 2, input: 3 } } }); + laneReporter.emit({ kind: 'test-result', payload: { testId: 'ok', title: 'ok', attempt: 0, status: 'passed', durationMs: 1 } }); + laneReporter.emit({ + kind: 'test-result', + payload: { testId: 'gap', title: 'gap', attempt: 0, status: 'unsupported', durationMs: 0, reason: 'Declared.', gap: 'quarantine' }, + }); + await laneReporter.finish(); + }); + + function rewriteLane(change: (result: DesktopRunResult) => void): void { + const file = path.join(laneReporter.root, 'run.json'); + const result = JSON.parse(fs.readFileSync(file, 'utf8')) as DesktopRunResult; + change(result); + fs.writeFileSync(file, JSON.stringify(result)); + } + + test('reads back the lane, the exclusions and each declared gap', async () => { + expect(await laneClient.query({ operation: 'status', runId: laneReporter.runId })).toMatchObject({ + terminal: true, + status: 'passed', + lane, + excluded: { platform: 1, scope: 2, input: 3 }, + counts: { unsupported: 1, quarantined: 1, declaredUnsupported: 0 }, + }); + expect(await laneClient.query({ operation: 'tests', runId: laneReporter.runId })).toMatchObject({ + items: [ + { title: 'ok', status: 'passed' }, + { title: 'gap', status: 'unsupported', gap: 'quarantine', message: 'Declared.' }, + ], + }); + }); + + test('a declared gap edited out of the retained report no longer reads as a passing lane', async () => { + rewriteLane((result) => { + delete result.tests[1].attempts[0].gap; + }); + + expect(await laneClient.query({ operation: 'status', runId: laneReporter.runId })).toMatchObject({ + status: 'incomplete', + counts: { unsupported: 1, quarantined: 0, declaredUnsupported: 0 }, + }); + }); + + test('reads a lane taken from the driver instance and rejects an unknown source', async () => { + rewriteLane((result) => { + Object.assign(result, { lane: { ...lane, source: 'driver' } }); + }); + expect(await laneClient.query({ operation: 'status', runId: laneReporter.runId })).toMatchObject({ + lane: { ...lane, source: 'driver' }, + }); + + rewriteLane((result) => { + Object.assign(result, { lane: { ...lane, source: 'cli' } }); + }); + await expect(laneClient.query({ operation: 'status', runId: laneReporter.runId })).rejects.toThrow(); + }); + + test.each([ + ['a lane that claims enforcement', (result: DesktopRunResult) => Object.assign(result, { lane: { ...lane, policyEnforced: true } })], + ['a physical lane with masking', (result: DesktopRunResult) => Object.assign(result, { lane: { ...lane, input: 'physical' } })], + [ + 'a negative exclusion count', + (result: DesktopRunResult) => Object.assign(result, { selection: { excluded: { platform: -1, scope: 0, input: 0 } } }), + ], + ['a gap on a passed attempt', (result: DesktopRunResult) => Object.assign(result.tests[0].attempts[0], { gap: 'unsupported' })], + ])('rejects %s in a retained report', async (_name, change) => { + rewriteLane((result) => { + change(result); + }); + + await expect(laneClient.query({ operation: 'status', runId: laneReporter.runId })).rejects.toThrow(); + }); + }); + test('snapshot persistence failures remain reporting failures after the writer recovers', async () => { const write = fs.writeFileSync; const injected = jest.spyOn(fs, 'writeFileSync').mockImplementation((file, ...args) => { diff --git a/packages/agentic/desktop-test/src/reporting/runQueries.ts b/packages/agentic/desktop-test/src/reporting/runQueries.ts index e75938f236..1f6843a2be 100644 --- a/packages/agentic/desktop-test/src/reporting/runQueries.ts +++ b/packages/agentic/desktop-test/src/reporting/runQueries.ts @@ -4,10 +4,10 @@ import { createHash, randomUUID } from 'node:crypto'; import { isNativeError } from 'node:util/types'; import { desktopArtifactPath } from './collector.js'; -import { DesktopReportedError, reportStatuses, redactDesktopSecrets } from './protocol.js'; -import type { TestStatus } from './protocol.js'; -import { summarizeDesktopRun } from './results.js'; -import type { DesktopRunResult } from './results.js'; +import { DesktopReportedError, reportStatuses, redactDesktopSecrets, selectionExclusionReasons, testGaps } from './protocol.js'; +import type { SelectionExclusions, TestGap, TestStatus } from './protocol.js'; +import { desktopInputLaneSources, summarizeDesktopRun } from './results.js'; +import type { DesktopInputLaneReport, DesktopRunResult, DesktopSelectionReport } from './results.js'; export type DesktopRunQuery = | { operation: 'status'; runId: string } @@ -20,11 +20,15 @@ export type DesktopRunStatus = { terminal: boolean; status: TestStatus; counts: DesktopRunResult['counts']; + lane?: DesktopInputLaneReport; + excluded?: SelectionExclusions; }; export type DesktopRunQueryItem = { id: string; title: string; status?: TestStatus; + /** Set on an `unsupported` test whose gap was declared in advance. */ + gap?: TestGap; message?: string; }; export type DesktopRunPage = { @@ -176,7 +180,15 @@ export class DesktopRunQueryService { }); if (result.status === 'failed') canonical.status = 'failed'; if (input.operation === 'status') - return { schemaVersion: 1, runId: run.runId, terminal, status: canonical.status, counts: canonical.counts }; + return { + schemaVersion: 1, + runId: run.runId, + terminal, + status: canonical.status, + counts: canonical.counts, + ...(canonical.lane ? { lane: canonical.lane } : {}), + ...(canonical.selection ? { excluded: canonical.selection.excluded } : {}), + }; let offset = 0; if (input.cursor) { const cursor = grant.cursors.get(input.cursor); @@ -225,6 +237,7 @@ export class DesktopRunQueryService { id: item.id, title: title.text, ...(item.status ? { status: item.status } : {}), + ...(item.gap ? { gap: item.gap } : {}), ...(message ? { message: message.text } : {}), }, omitted: title.omitted + (message?.omitted ?? 0), @@ -239,6 +252,7 @@ export class DesktopRunQueryService { id: id('test', test.testId), title: test.title, status: test.status, + ...(test.attempts.at(-1)?.gap ? { gap: test.attempts.at(-1)!.gap } : {}), message: test.attempts.at(-1)?.failure?.error.message ?? test.attempts.at(-1)?.reason, })); case 'work': @@ -346,11 +360,15 @@ function parseResult(text: string, runId: string): DesktopRunResult & { terminal throw new TypeError('Invalid persisted desktop attempt duration.'); const reason = attempt.reason; if (reason !== undefined) textField(reason); + const gap = attempt.gap; + if (gap !== undefined && (!testGaps.includes(gap as TestGap) || attempt.status !== 'unsupported')) + throw new TypeError('Invalid persisted desktop test gap.'); return { attempt: attempt.attempt, durationMs: attempt.durationMs, status: attempt.status, ...(typeof reason === 'string' ? { reason } : {}), + ...(gap !== undefined ? { gap: gap as TestGap } : {}), ...(attempt.failure !== undefined ? { failure: new DesktopReportedError(attempt.failure).failure } : {}), }; }); @@ -375,6 +393,8 @@ function parseResult(text: string, runId: string): DesktopRunResult & { terminal textField(value.finishedAt); if (typeof value.durationMs !== 'number' || !Number.isFinite(value.durationMs) || value.durationMs < 0) throw new TypeError('Invalid persisted desktop run duration.'); + const lane = laneField(value.lane); + const selection = selectionField(value.selection); const result = summarizeDesktopRun({ schemaVersion: 1, runId, @@ -382,6 +402,8 @@ function parseResult(text: string, runId: string): DesktopRunResult & { terminal startedAt: value.startedAt, finishedAt: value.finishedAt, durationMs: value.durationMs, + ...(lane ? { lane } : {}), + ...(selection ? { selection } : {}), tests, work, failures, @@ -390,6 +412,30 @@ function parseResult(text: string, runId: string): DesktopRunResult & { terminal return { ...result, status: value.status, ...(typeof value.terminal === 'boolean' ? { terminal: value.terminal } : {}) }; } +function laneField(value: unknown): DesktopInputLaneReport | undefined { + if (value === undefined) return undefined; + const lane = record(value); + const source = desktopInputLaneSources.find((candidate) => candidate === lane.source); + if (lane.policyEnforced !== false || source === undefined) throw new TypeError('Invalid persisted desktop input lane.'); + if (lane.input === 'physical' && lane.masking === undefined) return { input: 'physical', source, policyEnforced: false }; + if (lane.input === 'accessible' && (lane.masking === 'host-masked' || lane.masking === 'verified-masked')) + return { input: 'accessible', source, masking: lane.masking, policyEnforced: false }; + throw new TypeError('Invalid persisted desktop input lane.'); +} + +function selectionField(value: unknown): DesktopSelectionReport | undefined { + if (value === undefined) return undefined; + const excluded = record(record(value).excluded); + const counts = {} as SelectionExclusions; + for (const reason of selectionExclusionReasons) { + const count = excluded[reason]; + if (typeof count !== 'number' || !Number.isSafeInteger(count) || count < 0) + throw new TypeError('Invalid persisted desktop exclusion count.'); + counts[reason] = count; + } + return { excluded: counts }; +} + function record(value: unknown): Record { if (!value || typeof value !== 'object' || Array.isArray(value)) throw new TypeError('Expected a desktop run query object.'); return value as Record; diff --git a/packages/agentic/desktop-test/src/reporting/structured.test.ts b/packages/agentic/desktop-test/src/reporting/structured.test.ts index 1a7f5b1760..74a2c7f3d8 100644 --- a/packages/agentic/desktop-test/src/reporting/structured.test.ts +++ b/packages/agentic/desktop-test/src/reporting/structured.test.ts @@ -20,6 +20,24 @@ describe('structured desktop events', () => { expect(includeDesktopEvent(event, 'milestones', 'service')).toBe(true); }); + test('exclusion counts are progress detail and project as plain numbers', () => { + const event = makeEvent({ kind: 'selection', payload: { excluded: { platform: 1, scope: 0, input: 2 } } }); + + expect(includeDesktopEvent(event, 'milestones')).toBe(false); + expect(includeDesktopEvent(event, 'progress')).toBe(true); + expect(projectDesktopEvent(event).payload).toEqual({ excluded: { platform: 1, scope: 0, input: 2 } }); + }); + + test('a declared gap stays visible at every level and projects with its reason', () => { + const event = makeEvent({ + kind: 'test-result', + payload: { testId: 'test', title: 'gap', attempt: 0, status: 'unsupported', durationMs: 0, reason: 'Declared.', gap: 'quarantine' }, + }); + + expect(includeDesktopEvent(event, 'milestones')).toBe(true); + expect(projectDesktopEvent(event).payload).toMatchObject({ status: 'unsupported', gap: 'quarantine', reason: 'Declared.' }); + }); + test('projects and bounds large event text while retaining canonical IDs', () => { const event = makeEvent({ kind: 'output', diff --git a/packages/agentic/desktop-test/src/reporting/structured.ts b/packages/agentic/desktop-test/src/reporting/structured.ts index 8e89133ea6..544be9ebc9 100644 --- a/packages/agentic/desktop-test/src/reporting/structured.ts +++ b/packages/agentic/desktop-test/src/reporting/structured.ts @@ -41,6 +41,7 @@ export function includeDesktopEvent(event: DesktopEvent, level: DesktopEventLeve return event.payload.status !== 'passed'; case 'work-selected': case 'test-start': + case 'selection': case 'output': return false; } @@ -88,8 +89,12 @@ export function projectDesktopEvent(event: CollectedDesktopEvent): DesktopEventR payload.status = event.payload.status; payload.durationMs = event.payload.durationMs; if (event.payload.reason) payload.reason = clip(event.payload.reason); + if (event.payload.gap) payload.gap = event.payload.gap; if (event.payload.failure) payload.failure = projectFailure(event.payload.failure); break; + case 'selection': + payload.excluded = { ...event.payload.excluded }; + break; case 'artifact': payload.path = clip(event.payload.path); payload.description = clip(event.payload.description); diff --git a/packages/agentic/desktop-test/src/supervisor.test.ts b/packages/agentic/desktop-test/src/supervisor.test.ts new file mode 100644 index 0000000000..35ced5e4cf --- /dev/null +++ b/packages/agentic/desktop-test/src/supervisor.test.ts @@ -0,0 +1,71 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { accessibleLaneMaskedFeatures } from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; +import type { ResolvedDesktopTarget } from './config.js'; +import { startNativeDesktopService } from './driverService.js'; +import { DesktopRunReporter } from './reporting/collector.js'; +import { DesktopSupervisor } from './supervisor.js'; + +jest.mock('./driverService', () => ({ startDesktopService: jest.fn(), startNativeDesktopService: jest.fn() })); +jest.mock('./adapters', () => ({ resolveDesktopApplication: jest.fn(async () => ({ executablePath: 'Fixture.app' })) })); + +describe('desktop supervisor input lane', () => { + const platform = Object.getOwnPropertyDescriptor(process, 'platform')!; + const startNative = startNativeDesktopService as jest.MockedFunction; + let artifactsRoot: string; + + beforeEach(() => { + artifactsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-desktop-supervisor-')); + // The helper is mocked, so any host can stand in for the macOS machine the endpoint requires. + Object.defineProperty(process, 'platform', { ...platform, value: 'darwin' }); + startNative.mockReset(); + // eslint-disable-next-line @microsoft/sdl/no-insecure-url -- the mocked service stands in for a loopback Desktop Driver + startNative.mockResolvedValue({ url: 'http://127.0.0.1:1', close: async () => undefined } as Awaited< + ReturnType + >); + }); + + afterEach(() => { + Object.defineProperty(process, 'platform', platform); + fs.rmSync(artifactsRoot, { force: true, recursive: true }); + }); + + async function nativeServiceOptions(input?: DesktopInputLane) { + const output = { write: () => true }; + const reporter = new DesktopRunReporter({ artifactsRoot, errorOutput: output, output, reporterMode: 'plain', summary: false }); + const target: ResolvedDesktopTarget = { + endpoint: 'macos', + id: 'fixture', + instanceId: 'fixture-1', + projectRoot: artifactsRoot, + configPath: path.join(artifactsRoot, 'desktop.config.mts'), + host: { kind: 'installed' }, + }; + const supervisor = new DesktopSupervisor({ + target, + reporter, + reportSession: { reporter, delegated: true, environment: {}, close: async () => 0 }, + ...(input ? { input } : {}), + }); + try { + await supervisor.start(); + } finally { + await supervisor.close(); + } + expect(startNative).toHaveBeenCalledTimes(1); + return startNative.mock.calls[0][0]; + } + + test('starts the native host with physical input disabled for an accessible lane', async () => { + const options = await nativeServiceOptions('accessible'); + expect(options.disabledInputFeatures).toEqual([...accessibleLaneMaskedFeatures]); + }); + + test.each([['physical' as const], [undefined]])('leaves the native host unmasked for %s', async (input) => { + const options = await nativeServiceOptions(input); + expect(options).not.toHaveProperty('disabledInputFeatures'); + }); +}); diff --git a/packages/agentic/desktop-test/src/supervisor.ts b/packages/agentic/desktop-test/src/supervisor.ts index 4249c82ea6..eeb3766491 100644 --- a/packages/agentic/desktop-test/src/supervisor.ts +++ b/packages/agentic/desktop-test/src/supervisor.ts @@ -4,6 +4,8 @@ import path from 'node:path'; import { createHash } from 'node:crypto'; import type { DesktopDriverServer, DesktopHost } from '@fluentui-react-native/desktop-driver'; +import { accessibleLaneMaskedFeatures } from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; import type { DesktopAgentConnectionOptions } from './agentConnection.js'; import { startDesktopService, startNativeDesktopService } from './driverService.js'; import type { DesktopService, DesktopServiceOptions } from './driverService.js'; @@ -32,6 +34,11 @@ export type DesktopSupervisorOptions = { */ agent?: DesktopAgentConnectionOptions; recovery?: DesktopServiceOptions['recovery']; + /** + * The input lane of a `test` run. An accessible lane starts the native host with keyboard, physical click and wheel + * disabled. A host supplied through `hostFactory` is not masked here, so the suite verifies it before running anything. + */ + input?: DesktopInputLane; }; export class DesktopSupervisor { @@ -147,6 +154,7 @@ export class DesktopSupervisor { if (application.leaseNonce) this.reporter.addSecret(application.leaseNonce); return startNativeDesktopService({ ...options, + ...(this.options.input === 'accessible' ? { disabledInputFeatures: accessibleLaneMaskedFeatures } : {}), application, nativeDriver: this.target.nativeDriver, endpoint: this.target.endpoint, diff --git a/packages/agentic/desktop-test/src/wdio/configuration.ts b/packages/agentic/desktop-test/src/wdio/configuration.ts index 506294ad4f..f16768c542 100644 --- a/packages/agentic/desktop-test/src/wdio/configuration.ts +++ b/packages/agentic/desktop-test/src/wdio/configuration.ts @@ -2,7 +2,11 @@ import { pathToFileURL } from 'node:url'; import { randomUUID } from 'node:crypto'; import { installDesktopWebdriverCommands } from '@fluentui-react-native/desktop-driver/wdio'; import type { DesktopEndpoint, DesktopRenderer } from '@fluentui-react-native/desktop-driver'; +import { resolveDesktopClickMode } from '@fluentui-react-native/desktop-driver/authoring'; import type { DesktopReadiness } from '../config.js'; +import { desktopJasmineRegistry } from '../jasmine/registry.js'; +import { desktopTestSelectionFromEnvironment } from '../jasmine/selection.js'; +import { currentDesktopJasmineEnvironment, selectDesktopSpecs } from '../jasmine/specSelection.js'; import { connectReportProducer } from '../reporting/transport.js'; import { DesktopReportedError, serializeReportError } from '../reporting/protocol.js'; @@ -56,6 +60,8 @@ export async function createDesktopWdioConfiguration(settings: DesktopWdioSettin } const address = new URL(settings.url); const readiness = settings.readiness; + // The launcher read the selection too, so a malformed one fails the run before any worker starts. + const selection = desktopTestSelectionFromEnvironment(); const configuration = { ...hooks, runner: 'local', @@ -88,7 +94,7 @@ export async function createDesktopWdioConfiguration(settings: DesktopWdioSettin platformName: settings.endpoint === 'macos' ? 'macos' : 'windows', 'furn:target': settings.targetId, 'furn:launchMode': settings.ownership === 'external' ? 'attach' : 'launch', - 'furn:clickMode': 'auto', + 'furn:clickMode': resolveDesktopClickMode(undefined, 'auto', selection?.lane), }, ], beforeSession: async (...args: unknown[]) => { @@ -97,6 +103,10 @@ export async function createDesktopWdioConfiguration(settings: DesktopWdioSettin }, before: async (capabilities: unknown, specs: unknown, browser: WebdriverIO.Browser) => { installDesktopWebdriverCommands(browser, settings.url); + // Before the application is touched: a lane that cannot vouch for its tests must stop here. + if (settings.framework === 'jasmine') { + selectDesktopSpecs(desktopJasmineRegistry(), currentDesktopJasmineEnvironment(), browser.capabilities['furn:features']); + } if ((await browser.getWindowHandles()).length === 0) throw new Error('Native application did not expose a window.'); if (readiness?.testId) { await browser.$(`~${readiness.testId}`).waitForDisplayed({ diff --git a/packages/agentic/desktop-test/src/wdio/reporter.ts b/packages/agentic/desktop-test/src/wdio/reporter.ts index ab5b3df4e4..7379061b54 100644 --- a/packages/agentic/desktop-test/src/wdio/reporter.ts +++ b/packages/agentic/desktop-test/src/wdio/reporter.ts @@ -1,6 +1,8 @@ import { createHash, randomUUID } from 'node:crypto'; import WDIOReporter from '@wdio/reporter'; import type { HookStats, RunnerStats, TestStats } from '@wdio/reporter'; +import { desktopJasmineRegistry, excludedSpecCounts } from '../jasmine/registry.js'; +import type { DesktopSpecEntry } from '../jasmine/registry.js'; import { connectReportProducer } from '../reporting/transport.js'; import type { ReportProducer } from '../reporting/transport.js'; import { serializeReportError } from '../reporting/protocol.js'; @@ -71,6 +73,7 @@ export default class DesktopWdioReporter extends WDIOReporter { } override onTestStart(test: TestStats): void { + if (this.hidden(test)) return; this.report( { kind: 'test-start', payload: { testId: this.testId(test), title: test.fullTitle, attempt: this.attempt(test) } }, { ...this.context, test: test.uid }, @@ -90,13 +93,29 @@ export default class DesktopWdioReporter extends WDIOReporter { this.testResult(test); } + /** The spec the Jasmine adapter registered for a test, when `desktopIt` declared it. */ + private entry(test: TestStats): DesktopSpecEntry | undefined { + return desktopJasmineRegistry().entries.get(test.uid); + } + + /** An excluded case is not a test, and a blocked run reports its fault rather than every spec it disabled. */ + private hidden(test: TestStats): boolean { + return desktopJasmineRegistry().blocked || this.entry(test)?.kind === 'excluded'; + } + private testResult(test: TestStats): void { + if (this.hidden(test)) return; const testId = this.testId(test); const attempt = this.attempt(test); const key = `${testId}/${attempt}`; if (this.endedTests.has(key)) return; this.endedTests.add(key); - const status = test.state === 'pending' ? 'skipped' : test.state; + const entry = this.entry(test); + // A spec the adapter replaced or filtered out finishes as skipped or pending and never ran, whichever the framework calls it. + const notExecuted = test.state === 'pending' || test.state === 'skipped'; + const gap = notExecuted && entry?.kind === 'gap' ? entry.gap : undefined; + const missing = notExecuted && entry?.kind === 'run' ? entry.missing : undefined; + const status = gap || missing?.length ? 'unsupported' : test.state === 'pending' ? 'skipped' : test.state; this.report( { kind: 'test-result', @@ -106,6 +125,8 @@ export default class DesktopWdioReporter extends WDIOReporter { attempt, status, durationMs: test.duration, + ...(gap ? { gap: gap.kind, reason: gap.reason } : {}), + ...(missing?.length ? { reason: `Missing desktop requirements: ${missing.join(', ')}.` } : {}), ...(status === 'skipped' ? { reason: test.pendingReason || 'Skipped by the test framework.' } : {}), ...(status === 'failed' ? { failure: this.failure(`${key}/failure`, test.error ?? new AggregateError(test.errors ?? [], 'WDIO test failed.'), 'test') } @@ -143,6 +164,11 @@ export default class DesktopWdioReporter extends WDIOReporter { failure: this.failure(`${this.scopeId}/runner`, new Error(runner.error), 'infrastructure'), }, }); + const excluded = excludedSpecCounts(desktopJasmineRegistry()); + // Keyed by the spec group, so a retried attempt that announces the same exclusions again is counted once. + if (excluded.platform + excluded.scope + excluded.input > 0) { + this.report({ kind: 'selection', payload: { excluded, key: this.groupId } }); + } this.report({ kind: 'scope-end', payload: { status: runner.failures || runner.error ? 'failed' : 'passed' } }); this.schedule((producer) => producer.finish(runner.failures || runner.error ? 'failed' : 'passed')); } diff --git a/packages/agentic/desktop-test/src/wdio/run.ts b/packages/agentic/desktop-test/src/wdio/run.ts index c86eeb5efe..e9db1b7bbf 100644 --- a/packages/agentic/desktop-test/src/wdio/run.ts +++ b/packages/agentic/desktop-test/src/wdio/run.ts @@ -2,15 +2,32 @@ import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { resolve } from 'import-meta-resolve'; +import type { DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; import type { ResolvedDesktopConfig, DesktopWdioSuite } from '../config.js'; -import { resolveDesktopSpecs } from '../config.js'; +import { resolveDesktopSpecs, resolveDesktopTestScope } from '../config.js'; +import { desktopTestSelectionEnvironment, serializeDesktopTestSelection } from '../jasmine/selection.js'; import type { DesktopSupervisor } from '../supervisor.js'; import type { DesktopWdioSettings } from './configuration.js'; import { wdioGroupId } from './identity.js'; -export async function runDesktopWdioSuite(supervisor: DesktopSupervisor, config: ResolvedDesktopConfig, suiteName: string): Promise { +/** A lane selects tests through the Jasmine adapter, so a suite on another framework cannot honor one. */ +export function assertDesktopSuiteSupportsInput(suiteName: string, suite: DesktopWdioSuite, input: DesktopInputLane | undefined): void { + if (input && suite.framework !== 'jasmine') { + throw Object.assign(new TypeError(`--input ${input} requires a jasmine suite; "${suiteName}" uses ${suite.framework}.`), { + code: 'config', + }); + } +} + +export async function runDesktopWdioSuite( + supervisor: DesktopSupervisor, + config: ResolvedDesktopConfig, + suiteName: string, + input?: DesktopInputLane, +): Promise { const suite = config.config.suites?.[suiteName]; if (!suite || suite.kind !== 'wdio') throw new Error(`No standalone WDIO suite "${suiteName}" is configured.`); + assertDesktopSuiteSupportsInput(suiteName, suite, input); const specs = resolveDesktopSpecs(suite, supervisor.target.endpoint, config.projectRoot); const workIds: string[] = []; for (const selected of specs) { @@ -58,12 +75,19 @@ export async function runDesktopWdioSuite(supervisor: DesktopSupervisor, config: { mode: 0o600 }, ); const launcher = modulePath('@wdio/cli'); + // The selection travels as data in the environment: a worker parses it, and nothing is spliced into code that runs. + const selection = serializeDesktopTestSelection({ + endpoint: supervisor.target.endpoint, + ...(input ? { lane: input } : {}), + testScopes: config.config.testScopes, + values: resolveDesktopTestScope(config.testScopes, supervisor.target), + }); await supervisor.worker( { command: process.execPath, args: [fileURLToPath(new URL('../../config/launcher.cjs', import.meta.url)), pathToFileURL(launcher).href, configPath], cwd: config.projectRoot, - env: { E2ETEST_PLATFORM: supervisor.target.endpoint, NATIVE_TESTING: 'false' }, + env: { E2ETEST_PLATFORM: supervisor.target.endpoint, NATIVE_TESTING: 'false', [desktopTestSelectionEnvironment]: selection }, label: `WDIO ${suiteName} (${supervisor.target.endpoint})`, }, suiteDeadline(suite, specs.length), diff --git a/packages/agentic/native-core/references/contributing.md b/packages/agentic/native-core/references/contributing.md index abd2c0a237..f0e813a66a 100644 --- a/packages/agentic/native-core/references/contributing.md +++ b/packages/agentic/native-core/references/contributing.md @@ -55,6 +55,15 @@ full catalog before running the selected native tests. Add `--paper` to run the macOS Paper lane; a successful Paper build alone does not establish Paper runtime or input parity. +Executable cases are declared with `desktopIt` and its `requires` list (see +`@fluentui-react-native/storybook-desktop/references/story-tests.md`), not with run-time +`skip` guards. Cases that click or type declare `physical-click` and `keyboard`, which +derive the `physical` input lane; `focus` only asks the host to report focus support. +Only the placement cases declare no requirement and run in both lanes. The Win32 +Tab-exit behavior (#4377) is a declared `unsupported` gap, not a skip. Add +`--input accessible` or `--input physical` to select a lane: `accessible` selects only +the placement cases, and `physical` selects every case. + ## Apple boundary probe After building macOS Storybook, run the reproducible Apple boundary probe: diff --git a/packages/agentic/native-core/src/components/callout/Callout.stories.tsx b/packages/agentic/native-core/src/components/callout/Callout.stories.tsx index 2056a387f9..4010ef6020 100644 --- a/packages/agentic/native-core/src/components/callout/Callout.stories.tsx +++ b/packages/agentic/native-core/src/components/callout/Callout.stories.tsx @@ -4,6 +4,7 @@ import { Pressable, Text, View, StyleSheet } from 'react-native'; import type { Meta, StoryObj } from '@storybook/react-native'; import { useFocusablePressable } from '@fluentui-react-native/framework-base'; import type { PointerFocusMode } from '@fluentui-react-native/framework-base'; +import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; import type { WdioStory } from '@fluentui-react-native/storybook-desktop/testing'; import { Callout } from './Callout'; @@ -344,83 +345,78 @@ export default meta; type Story = WdioStory>; export const Default: Story = { tags: ['desktop-e2e'], - wdio: { - 'light-dismisses on an outside owner click and explicitly rearms': async ({ browser, expect, platform, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Light dismissal requires physical owner input.'); - return; - } - const status = await browser.$('~modern-callout-status'); - const dismissal = await browser.$('~modern-callout-dismissal'); - // The App SDK notification itself has no cause; its unclassified path can win the outside-pointer race. - const expected = platform === 'windows' ? /^Dismissed: (outside-pointer|native-light-dismiss)$/ : 'Dismissed: outside-pointer'; - for (let attempt = 0; attempt < 2; attempt++) { + wdio: [ + desktopIt( + 'light-dismisses on an outside owner click and explicitly rearms', + { requires: ['physical-click'] }, + async ({ browser, expect, platform }) => { + const status = await browser.$('~modern-callout-status'); + const dismissal = await browser.$('~modern-callout-dismissal'); + // The App SDK notification itself has no cause; its unclassified path can win the outside-pointer race. + const expected = platform === 'windows' ? /^Dismissed: (outside-pointer|native-light-dismiss)$/ : 'Dismissed: outside-pointer'; + for (let attempt = 0; attempt < 2; attempt++) { + await expect(status).toHaveText('Ready'); + await expect(dismissal).toHaveText('Open'); + await status.click(); + await expect(dismissal).toHaveText(expected); + // A latched presentation must not reopen merely because open remains true; readiness would reset the label. + await browser.pause(500); + await expect(dismissal).toHaveText(expected); + await (await browser.$('~modern-callout-reopen')).click(); + } await expect(status).toHaveText('Ready'); await expect(dismissal).toHaveText('Open'); - await status.click(); - await expect(dismissal).toHaveText(expected); - // A latched presentation must not reopen merely because open remains true; readiness would reset the label. + }, + ), + desktopIt( + 'keeps the presentation open for pointer input on its view anchor', + { requires: ['physical-click'] }, + async ({ browser, expect, platform }) => { + await expect(await browser.$('~modern-callout-status')).toHaveText('Ready'); + const dismissal = await browser.$('~modern-callout-dismissal'); + await expect(dismissal).toHaveText('Open'); + await (await browser.$('~modern-callout-anchor')).click(); await browser.pause(500); - await expect(dismissal).toHaveText(expected); - await (await browser.$('~modern-callout-reopen')).click(); - } - await expect(status).toHaveText('Ready'); - await expect(dismissal).toHaveText('Open'); - }, - 'keeps the presentation open for pointer input on its view anchor': async ({ browser, expect, platform, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Anchor input requires physical owner input.'); - return; - } - await expect(await browser.$('~modern-callout-status')).toHaveText('Ready'); - const dismissal = await browser.$('~modern-callout-dismissal'); - await expect(dismissal).toHaveText('Open'); - await (await browser.$('~modern-callout-anchor')).click(); - await browser.pause(500); - await expect(dismissal).toHaveText('Open'); - await (await browser.$('~modern-callout-status')).click(); - await expect(dismissal).toHaveText( - platform === 'windows' ? /^Dismissed: (outside-pointer|native-light-dismiss)$/ : 'Dismissed: outside-pointer', - ); - }, - 'places content below the view anchor with the requested gap': async (context) => { + await expect(dismissal).toHaveText('Open'); + await (await browser.$('~modern-callout-status')).click(); + await expect(dismissal).toHaveText( + platform === 'windows' ? /^Dismissed: (outside-pointer|native-light-dismiss)$/ : 'Dismissed: outside-pointer', + ); + }, + ), + desktopIt('places content below the view anchor with the requested gap', async (context) => { const { expectCalloutPlacement } = await import('./placement.wdio.ts'); await expectCalloutPlacement(context); - }, - }, + }), + ], }; export const NativeOperations: Story = { args: { exercise: 'commands' }, tags: ['desktop-e2e'], - wdio: { - 'readies, confirms descendant focus, closes, and explicitly rearms': async ({ browser, expect, platform, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Physical owner input unavailable.'); - return; - } - const status = await browser.$('~modern-callout-status'); - const operations = await browser.$('~modern-callout-operations'); - await expect(status).toHaveText('Ready'); - for (let attempt = 0; attempt < 2; attempt++) { - await (await browser.$('~modern-callout-reopen')).click(); - await expect(operations).toHaveText('confirmed/confirmed/confirmed'); - await expect(await browser.$('~modern-callout-dismissal')).toHaveText('Dismissed: programmatic'); - await expect(await browser.$('~modern-callout-focus-status')).toHaveText('Child focused'); - // Windows popups are owned tool windows that the driver does not enumerate as top-level windows. - if (platform === 'macos') await browser.waitUntil(async () => (await browser.getWindowHandles()).length === 1, { timeout: 5000 }); - } - }, - }, + wdio: [ + desktopIt( + 'readies, confirms descendant focus, closes, and explicitly rearms', + { requires: ['physical-click'] }, + async ({ browser, expect, platform }) => { + const status = await browser.$('~modern-callout-status'); + const operations = await browser.$('~modern-callout-operations'); + await expect(status).toHaveText('Ready'); + for (let attempt = 0; attempt < 2; attempt++) { + await (await browser.$('~modern-callout-reopen')).click(); + await expect(operations).toHaveText('confirmed/confirmed/confirmed'); + await expect(await browser.$('~modern-callout-dismissal')).toHaveText('Dismissed: programmatic'); + await expect(await browser.$('~modern-callout-focus-status')).toHaveText('Child focused'); + // Windows popups are owned tool windows that the driver does not enumerate as top-level windows. + if (platform === 'macos') await browser.waitUntil(async () => (await browser.getWindowHandles()).length === 1, { timeout: 5000 }); + } + }, + ), + ], }; export const KeyboardDismissal: Story = { args: { exercise: 'focus' }, - wdio: { - 'dismisses focused content on Escape': async ({ browser, expect, skip }) => { - const features = browser.capabilities['furn:features']; - if (!features?.physicalClick || !features.keyboard) { - skip('Escape dismissal requires physical owner and keyboard input.'); - return; - } + wdio: [ + desktopIt('dismisses focused content on Escape', { requires: ['physical-click', 'keyboard'] }, async ({ browser, expect }) => { const status = await browser.$('~modern-callout-status'); await expect(status).toHaveText('Ready'); await (await browser.$('~modern-callout-reopen')).click(); @@ -430,195 +426,186 @@ export const KeyboardDismissal: Story = { await expect(dismissal).toHaveText('Open'); await browser.keys('\uE00C'); await expect(dismissal).toHaveText('Dismissed: escape'); - }, - }, + }), + ], }; export const FirstFocusable: Story = { render: () => , tags: ['desktop-e2e'], - wdio: { - 'confirms the first eligible content without a FocusZone': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Entry focus requires physical owner activation.'); - return; - } + wdio: [ + desktopIt('confirms the first eligible content without a FocusZone', { requires: ['physical-click'] }, async ({ browser, expect }) => { await (await browser.$('~entry-callout-open')).click(); await expect(await browser.$('~entry-callout-status')).toHaveText('confirmed'); await expect(await browser.$('~entry-callout-focused')).toHaveText('First'); - }, - }, + }), + ], }; export const LastFocusable: Story = { render: () => , tags: ['desktop-e2e'], - wdio: { - 'confirms the last eligible nested content without a FocusZone': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Entry focus requires physical owner activation.'); - return; - } - await (await browser.$('~entry-callout-open')).click(); - await expect(await browser.$('~entry-callout-status')).toHaveText('confirmed'); - await expect(await browser.$('~entry-callout-focused')).toHaveText('Last'); - }, - }, + wdio: [ + desktopIt( + 'confirms the last eligible nested content without a FocusZone', + { requires: ['physical-click'] }, + async ({ browser, expect }) => { + await (await browser.$('~entry-callout-open')).click(); + await expect(await browser.$('~entry-callout-status')).toHaveText('confirmed'); + await expect(await browser.$('~entry-callout-focused')).toHaveText('Last'); + }, + ), + ], }; export const EmptyFocus: Story = { render: () => , tags: ['desktop-e2e'], - wdio: { - 'reports missing eligible content instead of manufacturing focus confirmation': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Entry focus requires physical owner activation.'); - return; - } - await (await browser.$('~entry-callout-open')).click(); - await expect(await browser.$('~entry-callout-status')).toHaveText('not-mounted'); - await expect(await browser.$('~entry-callout-focused')).toHaveText('None'); - }, - }, + wdio: [ + desktopIt( + 'reports missing eligible content instead of manufacturing focus confirmation', + { requires: ['physical-click'] }, + async ({ browser, expect }) => { + await (await browser.$('~entry-callout-open')).click(); + await expect(await browser.$('~entry-callout-status')).toHaveText('not-mounted'); + await expect(await browser.$('~entry-callout-focused')).toHaveText('None'); + }, + ), + ], }; export const ContainerFocus: Story = { render: () => , tags: ['desktop-e2e'], - wdio: { - 'confirms an explicitly focusable empty content container': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Container focus requires physical owner activation.'); - return; - } + wdio: [ + desktopIt('confirms an explicitly focusable empty content container', { requires: ['physical-click'] }, async ({ browser, expect }) => { await (await browser.$('~entry-callout-open')).click(); await expect(await browser.$('~entry-callout-status')).toHaveText('confirmed'); await expect(await browser.$('~entry-callout-focused')).toHaveText('Container'); - }, - }, + }), + ], }; export const DeferredContainerFocus: Story = { render: () => , tags: ['desktop-e2e'], - wdio: { - 'orders a focus request after the content update committed before it': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Container focus requires physical owner activation.'); - return; - } - await (await browser.$('~entry-callout-open')).click(); - await expect(await browser.$('~entry-callout-status')).toHaveText('confirmed'); - await expect(await browser.$('~entry-callout-focused')).toHaveText('Container'); - }, - }, + wdio: [ + desktopIt( + 'orders a focus request after the content update committed before it', + { requires: ['physical-click'] }, + async ({ browser, expect }) => { + await (await browser.$('~entry-callout-open')).click(); + await expect(await browser.$('~entry-callout-status')).toHaveText('confirmed'); + await expect(await browser.$('~entry-callout-focused')).toHaveText('Container'); + }, + ), + ], }; export const LocalRectangle: Story = { args: { anchorKind: 'rect' }, - wdio: { - 'places content below the local rectangle with the requested gap': async (context) => { + wdio: [ + desktopIt('places content below the local rectangle with the requested gap', async (context) => { const { expectCalloutPlacement } = await import('./placement.wdio.ts'); await expectCalloutPlacement(context, { x: 40, y: 30, height: 30 }); - }, - }, + }), + ], }; export const LocalPoint: Story = { args: { anchorKind: 'point' }, - wdio: { - 'places content below the local point with the requested gap': async (context) => { + wdio: [ + desktopIt('places content below the local point with the requested gap', async (context) => { const { expectCalloutPlacement } = await import('./placement.wdio.ts'); await expectCalloutPlacement(context, { x: 40, y: 60, height: 0 }); - }, - }, + }), + ], }; export const MenuComposition: Story = { render: ({ pointerFocus }) => , args: { pointerFocus: 'platform' }, argTypes: { pointerFocus: { control: 'select', options: ['platform', 'always', 'never'] } }, tags: ['desktop-e2e'], - wdio: { - 'enters a FocusZone, navigates by arrow, and follows the platform focus convention on Escape': async ({ - browser, - expect, - platform, - skip, - }) => { - const { exerciseMenuComposition } = await import('./menuComposition.wdio.ts'); - // Windows and Win32 focus a pressed trigger, so closing the menu restores it. AppKit leaves keyboard focus where - // it was on click, so macOS restores the owner window's previous first responder instead. - await exerciseMenuComposition({ browser, expect, skip }, { restoresTrigger: platform !== 'macos' }); - }, - }, + wdio: [ + desktopIt( + 'enters a FocusZone, navigates by arrow, and follows the platform focus convention on Escape', + { requires: ['physical-click', 'keyboard', 'focus'] }, + async ({ browser, expect, platform }) => { + const { exerciseMenuComposition } = await import('./menuComposition.wdio.ts'); + // Windows and Win32 focus a pressed trigger, so closing the menu restores it. AppKit leaves keyboard focus where + // it was on click, so macOS restores the owner window's previous first responder instead. + await exerciseMenuComposition({ browser, expect }, { restoresTrigger: platform !== 'macos' }); + }, + ), + ], }; export const TabContinuation: Story = { render: () => , tags: ['desktop-e2e'], - wdio: { - 'closes on Tab and continues past the anchor, and on Shift+Tab returns to it': async ({ browser, expect, platform, skip }) => { - const features = browser.capabilities['furn:features']; - if (platform === 'win32') { - skip('Win32 hosts keep the cycle behavior (#4377).'); - return; - } - if (!features?.physicalClick || !features.keyboard || !features.focus) { - skip('Tab exit requires physical owner input, keyboard input, and focus observation.'); - return; - } - const trigger = await browser.$('~modern-menu-trigger'); - const status = await browser.$('~modern-menu-status'); - await trigger.click(); - await expect(status).toHaveText('Ready: confirmed'); - await browser.keys('\uE004'); - await expect(status).toHaveText('Dismissed: tab-forward'); - await browser.waitUntil(async () => (await (await browser.$('~modern-menu-after')).getProperty('focused')) === true, { - timeout: 5000, - timeoutMsg: 'Expected Tab to continue to the view after the anchor.', - }); - await trigger.click(); - await expect(status).toHaveText('Ready: confirmed'); - await browser.keys(['\uE008', '\uE004']); - await expect(status).toHaveText('Dismissed: tab-backward'); - await browser.waitUntil(async () => (await trigger.getProperty('focused')) === true, { - timeout: 5000, - timeoutMsg: 'Expected Shift+Tab to focus the anchor.', - }); - }, - }, + wdio: [ + desktopIt( + 'closes on Tab and continues past the anchor, and on Shift+Tab returns to it', + { + requires: ['physical-click', 'keyboard', 'focus'], + unsupported: { platforms: ['win32'], reason: 'Win32 hosts keep the cycle behavior.', issue: '#4377' }, + }, + async ({ browser, expect }) => { + const trigger = await browser.$('~modern-menu-trigger'); + const status = await browser.$('~modern-menu-status'); + await trigger.click(); + await expect(status).toHaveText('Ready: confirmed'); + await browser.keys('\uE004'); + await expect(status).toHaveText('Dismissed: tab-forward'); + await browser.waitUntil(async () => (await (await browser.$('~modern-menu-after')).getProperty('focused')) === true, { + timeout: 5000, + timeoutMsg: 'Expected Tab to continue to the view after the anchor.', + }); + await trigger.click(); + await expect(status).toHaveText('Ready: confirmed'); + await browser.keys(['\uE008', '\uE004']); + await expect(status).toHaveText('Dismissed: tab-backward'); + await browser.waitUntil(async () => (await trigger.getProperty('focused')) === true, { + timeout: 5000, + timeoutMsg: 'Expected Shift+Tab to focus the anchor.', + }); + }, + ), + ], }; export const TabDismissal: Story = { render: () => , tags: ['desktop-e2e'], - wdio: { - 'closes on Tab and leaves the owner window its retained focus': async ({ browser, expect, platform, skip }) => { - const features = browser.capabilities['furn:features']; - if (platform === 'win32') { - skip('Win32 hosts keep the cycle behavior (#4377).'); - return; - } - if (!features?.physicalClick || !features.keyboard || !features.focus) { - skip('Tab exit requires physical owner input, keyboard input, and focus observation.'); - return; - } - const trigger = await browser.$('~modern-menu-trigger'); - const status = await browser.$('~modern-menu-status'); - await trigger.click(); - await expect(status).toHaveText('Ready: confirmed'); - await browser.keys('\uE004'); - await expect(status).toHaveText('Dismissed: tab-forward'); - await browser.waitUntil(async () => (await trigger.getProperty('focused')) === true, { - timeout: 5000, - timeoutMsg: 'Expected the owner window to keep the focused trigger.', - }); - expect(await (await browser.$('~modern-menu-after')).getProperty('focused')).toBe(false); - }, - }, + wdio: [ + desktopIt( + 'closes on Tab and leaves the owner window its retained focus', + { + requires: ['physical-click', 'keyboard', 'focus'], + unsupported: { platforms: ['win32'], reason: 'Win32 hosts keep the cycle behavior.', issue: '#4377' }, + }, + async ({ browser, expect }) => { + const trigger = await browser.$('~modern-menu-trigger'); + const status = await browser.$('~modern-menu-status'); + await trigger.click(); + await expect(status).toHaveText('Ready: confirmed'); + await browser.keys('\uE004'); + await expect(status).toHaveText('Dismissed: tab-forward'); + await browser.waitUntil(async () => (await trigger.getProperty('focused')) === true, { + timeout: 5000, + timeoutMsg: 'Expected the owner window to keep the focused trigger.', + }); + expect(await (await browser.$('~modern-menu-after')).getProperty('focused')).toBe(false); + }, + ), + ], }; export const MenuCompositionFocusingTrigger: Story = { render: ({ pointerFocus }) => , args: { pointerFocus: 'always' }, argTypes: MenuComposition.argTypes, tags: ['desktop-e2e'], - wdio: { - 'restores keyboard focus to a trigger focused on press': async ({ browser, expect, skip }) => { - const { exerciseMenuComposition } = await import('./menuComposition.wdio.ts'); - await exerciseMenuComposition({ browser, expect, skip }, { restoresTrigger: true }); - }, - }, + wdio: [ + desktopIt( + 'restores keyboard focus to a trigger focused on press', + { requires: ['physical-click', 'keyboard', 'focus'] }, + async ({ browser, expect }) => { + const { exerciseMenuComposition } = await import('./menuComposition.wdio.ts'); + await exerciseMenuComposition({ browser, expect }, { restoresTrigger: true }); + }, + ), + ], }; const styles = StyleSheet.create({ story: { padding: 24, gap: 16 }, diff --git a/packages/agentic/native-core/src/components/callout/menuComposition.wdio.ts b/packages/agentic/native-core/src/components/callout/menuComposition.wdio.ts index ecf57e52c0..38f8554eea 100644 --- a/packages/agentic/native-core/src/components/callout/menuComposition.wdio.ts +++ b/packages/agentic/native-core/src/components/callout/menuComposition.wdio.ts @@ -2,18 +2,14 @@ import type { WdioStoryContext } from '@fluentui-react-native/storybook-desktop/ /** * Opens the menu by pointer, enters its FocusZone, navigates by arrow with wrapping, and closes it with Escape, twice. - * `restoresTrigger` states whether the pressed trigger took keyboard focus, which decides whether the owner window - * returns focus to it when the menu closes. + * The calling case declares the physical click, keyboard and focus observation it needs. `restoresTrigger` states + * whether the pressed trigger took keyboard focus, which decides whether the owner window returns focus to it when the + * menu closes. */ export async function exerciseMenuComposition( - { browser, expect, skip }: Pick, + { browser, expect }: Pick, { restoresTrigger }: { restoresTrigger: boolean }, ): Promise { - const features = browser.capabilities['furn:features']; - if (!features?.physicalClick || !features.keyboard || !features.focus) { - skip('Menu composition requires physical owner input, keyboard input, and focus observation.'); - return; - } const trigger = await browser.$('~modern-menu-trigger'); const status = await browser.$('~modern-menu-status'); const focused = await browser.$('~modern-menu-focus'); diff --git a/packages/agentic/native-core/src/components/focus-zone/FocusZone.stories.tsx b/packages/agentic/native-core/src/components/focus-zone/FocusZone.stories.tsx index 9614a69b70..b9fd07ca76 100644 --- a/packages/agentic/native-core/src/components/focus-zone/FocusZone.stories.tsx +++ b/packages/agentic/native-core/src/components/focus-zone/FocusZone.stories.tsx @@ -2,6 +2,7 @@ import * as React from 'react'; import { Pressable, Text, View, StyleSheet } from 'react-native'; import type { Meta, StoryObj } from '@storybook/react-native'; +import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; import type { WdioStory } from '@fluentui-react-native/storybook-desktop/testing'; import { FocusZone } from './FocusZone'; @@ -150,71 +151,63 @@ export default meta; type Story = WdioStory>; export const Default: Story = { tags: ['desktop-e2e'], - wdio: { - 'confirms native default, edge, and specific descendant requests': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Native commands require physical input for their owner controls.'); - return; - } - for (const [strategy, item] of [ - ['default', 2], - ['first', 1], - ['last', 3], - ['default', 2], - ['target', 2], - ['first', 1], - ] as const) { - await (await browser.$(`~modern-focus-request-${strategy}`)).click(); - await expect(await browser.$('~modern-focus-status')).toHaveText('confirmed'); - await browser.waitUntil(async () => (await (await browser.$(`~modern-focus-item-${item}`)).getProperty('focused')) === true, { - timeout: 5000, - timeoutMsg: `Expected actual focus on item ${item}.`, - }); - } - }, - }, + wdio: [ + desktopIt( + 'confirms native default, edge, and specific descendant requests', + { requires: ['physical-click'] }, + async ({ browser, expect }) => { + for (const [strategy, item] of [ + ['default', 2], + ['first', 1], + ['last', 3], + ['default', 2], + ['target', 2], + ['first', 1], + ] as const) { + await (await browser.$(`~modern-focus-request-${strategy}`)).click(); + await expect(await browser.$('~modern-focus-status')).toHaveText('confirmed'); + await browser.waitUntil(async () => (await (await browser.$(`~modern-focus-item-${item}`)).getProperty('focused')) === true, { + timeout: 5000, + timeoutMsg: `Expected actual focus on item ${item}.`, + }); + } + }, + ), + ], }; export const Disabled: Story = { args: { disabled: true }, - wdio: { - 'refuses a disabled-zone request': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Physical input unavailable.'); - return; - } + wdio: [ + desktopIt('refuses a disabled-zone request', { requires: ['physical-click'] }, async ({ browser, expect }) => { await (await browser.$('~modern-focus-request-first')).click(); await expect(await browser.$('~modern-focus-status')).toHaveText('not-focusable'); - }, - }, + }), + ], }; export const TargetLifetimes: Story = { render: () => , - wdio: { - 'rejects unfocusable and outside live targets': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Physical input unavailable.'); - return; - } + wdio: [ + desktopIt('rejects unfocusable and outside live targets', { requires: ['physical-click'] }, async ({ browser, expect }) => { await (await browser.$('~modern-focus-request-unfocusable')).click(); await expect(await browser.$('~modern-focus-status')).toHaveText('not-focusable'); await (await browser.$('~modern-focus-request-outside')).click(); await expect(await browser.$('~modern-focus-status')).toHaveText('not-mounted'); expect(await (await browser.$('~modern-focus-outside')).getProperty('focused')).toBe(false); - }, - 'cancels pre-aborted requests and invalidates detached commands': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Physical input unavailable.'); - return; - } - await (await browser.$('~modern-focus-request-cancelled')).click(); - await expect(await browser.$('~modern-focus-status')).toHaveText('cancelled'); - expect(await (await browser.$('~modern-focus-item-1')).getProperty('focused')).toBe(false); - await (await browser.$('~modern-focus-unmount')).click(); - await expect(await browser.$('~modern-focus-zone')).not.toExist(); - await (await browser.$('~modern-focus-request-detached')).click(); - await expect(await browser.$('~modern-focus-status')).toHaveText('not-mounted'); - }, - }, + }), + desktopIt( + 'cancels pre-aborted requests and invalidates detached commands', + { requires: ['physical-click'] }, + async ({ browser, expect }) => { + await (await browser.$('~modern-focus-request-cancelled')).click(); + await expect(await browser.$('~modern-focus-status')).toHaveText('cancelled'); + expect(await (await browser.$('~modern-focus-item-1')).getProperty('focused')).toBe(false); + await (await browser.$('~modern-focus-unmount')).click(); + await expect(await browser.$('~modern-focus-zone')).not.toExist(); + await (await browser.$('~modern-focus-request-detached')).click(); + await expect(await browser.$('~modern-focus-status')).toHaveText('not-mounted'); + }, + ), + ], }; const styles = StyleSheet.create({ story: { padding: 24, gap: 16 }, diff --git a/packages/agentic/native-core/src/legacy/callout/Callout.stories.tsx b/packages/agentic/native-core/src/legacy/callout/Callout.stories.tsx index 2a4d0b6e8b..864f43add9 100644 --- a/packages/agentic/native-core/src/legacy/callout/Callout.stories.tsx +++ b/packages/agentic/native-core/src/legacy/callout/Callout.stories.tsx @@ -2,6 +2,7 @@ import * as React from 'react'; import { Pressable, StyleSheet, Text, View } from 'react-native'; +import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; import type { WdioStory } from '@fluentui-react-native/storybook-desktop/testing'; import type { Meta, StoryObj } from '@storybook/react-native'; @@ -185,32 +186,31 @@ type Story = WdioStory>; export const Default: Story = { tags: ['desktop-e2e'], render: (args) => , - wdio: { - 'opens, light-dismisses, and recreates a native popup': async ({ browser, expect, platform, skip }) => { - const features = browser.capabilities['furn:features']; - if (!features?.physicalClick) { - skip('Popup lifecycle qualification requires physical pointer input.'); - return; - } - const status = await browser.$('~agentic-storybook-callout-status'); - for (let attempt = 0; attempt < 2; attempt++) { - await expect(status).toHaveText('Native window: Shown'); - // Only the AppKit provider enumerates the popup as a top-level window. - if (platform === 'macos') expect(await browser.getWindowHandles()).toHaveLength(2); - await status.click(); - if (platform === 'macos') { - await browser.waitUntil(async () => (await browser.getWindowHandles()).length === 1, { - timeout: 5000, - timeoutMsg: 'Expected dismissal to remove the native Callout window.', - }); - } - await expect(status).toHaveText('Native window: Dismissed'); - if (attempt === 0) { - await (await browser.$('~agentic-storybook-callout-trigger')).click(); + wdio: [ + desktopIt( + 'opens, light-dismisses, and recreates a native popup', + { requires: ['physical-click'] }, + async ({ browser, expect, platform }) => { + const status = await browser.$('~agentic-storybook-callout-status'); + for (let attempt = 0; attempt < 2; attempt++) { + await expect(status).toHaveText('Native window: Shown'); + // Only the AppKit provider enumerates the popup as a top-level window. + if (platform === 'macos') expect(await browser.getWindowHandles()).toHaveLength(2); + await status.click(); + if (platform === 'macos') { + await browser.waitUntil(async () => (await browser.getWindowHandles()).length === 1, { + timeout: 5000, + timeoutMsg: 'Expected dismissal to remove the native Callout window.', + }); + } + await expect(status).toHaveText('Native window: Dismissed'); + if (attempt === 0) { + await (await browser.$('~agentic-storybook-callout-trigger')).click(); + } } - } - }, - }, + }, + ), + ], }; export const Placement: Story = { @@ -230,19 +230,18 @@ export const WindowCommands: Story = { }, render: (args) => , tags: ['desktop-e2e'], - wdio: { - 'dismisses initially focused content on Escape': async ({ browser, expect, skip }) => { - const features = browser.capabilities['furn:features']; - if (!features?.physicalClick || !features.keyboard) { - skip('Escape dismissal requires an interactive desktop with keyboard input.'); - return; - } - const status = await browser.$('~agentic-storybook-callout-status'); - await expect(status).toHaveText('Native window: Shown'); - await browser.keys('\uE00C'); - await expect(status).toHaveText('Native window: Dismissed'); - }, - }, + wdio: [ + desktopIt( + 'dismisses initially focused content on Escape', + { requires: ['physical-click', 'keyboard'] }, + async ({ browser, expect }) => { + const status = await browser.$('~agentic-storybook-callout-status'); + await expect(status).toHaveText('Native window: Shown'); + await browser.keys('\uE00C'); + await expect(status).toHaveText('Native window: Dismissed'); + }, + ), + ], parameters: { docs: { description: { diff --git a/packages/agentic/storybook-desktop/config/server.cjs b/packages/agentic/storybook-desktop/config/server.cjs index 99421e224d..f4c461e76b 100644 --- a/packages/agentic/storybook-desktop/config/server.cjs +++ b/packages/agentic/storybook-desktop/config/server.cjs @@ -113,6 +113,11 @@ async function startDesktopStorybookServer({ keepAlive = setInterval(() => undefined, 60_000); if (driverManifest) { const { StorybookChannelOrchestrator } = await import('../lib/driver/index.js'); + // An accessible driver cannot perform physical input, so every test that attaches to it runs the accessible lane. + const maskedFeatures = + driverManifest.wdio?.input === 'accessible' + ? (await import('@fluentui-react-native/desktop-driver/authoring')).accessibleLaneMaskedFeatures + : undefined; const target = { endpoint: driverManifest.endpoint, id: driverManifest.targetId, @@ -130,6 +135,7 @@ async function startDesktopStorybookServer({ platformName: target.platformName, storyRootTestId: target.storyRootTestId, windows: createFakeStoryWindows(driverManifest.storyManifest, target.storyRootTestId), + ...(maskedFeatures ? { features: Object.fromEntries(maskedFeatures.map((feature) => [feature, false])) } : {}), }); driver = await createDesktopDriverServer({ host, @@ -149,6 +155,7 @@ async function startDesktopStorybookServer({ onStderr: (text) => report.reporter.emit({ kind: 'output', payload: { stream: 'stderr', text } }, 'native-helper', { service: 'native-helper' }), onFailure: fatal, + ...(maskedFeatures ? { disabledInputFeatures: maskedFeatures } : {}), ...(agent ? { agent: { descriptor: agent.descriptor, ...(agent.root === undefined ? {} : { root: agent.root }) }, diff --git a/packages/agentic/storybook-desktop/config/storybook-control.cjs b/packages/agentic/storybook-desktop/config/storybook-control.cjs index 40df0e989b..2e1f1aa09c 100644 --- a/packages/agentic/storybook-desktop/config/storybook-control.cjs +++ b/packages/agentic/storybook-desktop/config/storybook-control.cjs @@ -145,8 +145,12 @@ async function runAuthoredTests() { const configUrl = pathToFileURL(path.join(__dirname, '..', 'lib', 'config', 'index.js')).href; const { formatDesktopStorybookSmokeTestSummary, runDesktopStorybookSmokeTests } = await import(smokeTestsUrl); const { makeDesktopStorybookConfig } = await import(configUrl); + // The driver instance's lane is a per-run choice the CLI resolved, so it travels as an option, not as configuration. + const { input, ...wdio } = manifest.wdio ?? {}; const result = await runDesktopStorybookSmokeTests({ - config: makeDesktopStorybookConfig({ projectRoot: process.cwd(), wdio: manifest.wdio }), + config: makeDesktopStorybookConfig({ projectRoot: process.cwd(), wdio }), + ...(input ? { input } : {}), + ...(manifest.scope ? { scope: manifest.scope } : {}), // eslint-disable-next-line @microsoft/sdl/no-insecure-url -- the Desktop Driver is loopback-only driverUrl: `http://127.0.0.1:${manifest.driverPort}`, manifest: manifest.storyManifest, diff --git a/packages/agentic/storybook-desktop/config/transform-story-tests.cjs b/packages/agentic/storybook-desktop/config/transform-story-tests.cjs index bc47137db0..ae64781038 100644 --- a/packages/agentic/storybook-desktop/config/transform-story-tests.cjs +++ b/packages/agentic/storybook-desktop/config/transform-story-tests.cjs @@ -242,6 +242,136 @@ function collectWdioTests(program) { return tests; } +// The physical requirement each WebDriver action source type implies, in the order messages list them. +const actionSourceRequirements = { key: 'keyboard', pointer: 'physical-click', wheel: 'wheel' }; +const requirementInputKinds = { keyboard: 'keyboard', 'physical-click': 'pointer', wheel: 'wheel' }; +const valueCommands = new Set(['setValue', 'addValue', 'clearValue']); + +function memberName(callee) { + const property = callee.node.property; + if (!callee.node.computed) return types.isIdentifier(property) ? property.name : undefined; + if (types.isStringLiteral(property)) return property.value; + return types.isTemplateLiteral(property) && property.expressions.length === 0 ? property.quasis[0].value.cooked : undefined; +} + +function isBrowser(object) { + object = unwrap(object); + if (object.isIdentifier({ name: 'browser' })) return true; + return (object.isMemberExpression() || object.isOptionalMemberExpression()) && memberName(object) === 'browser'; +} + +/** Follows a constant to the value it holds, so a payload declared above the call is still read. */ +function readConstant(value, seen = new Set()) { + value = unwrap(value); + if (value.isIdentifier()) { + const binding = value.scope.getBinding(value.node.name); + const declarator = binding?.path; + if ( + binding?.constant && + declarator?.isVariableDeclarator() && + declarator.node.init && + declarator.get('id').isIdentifier() && + !seen.has(binding) + ) { + seen.add(binding); + return readConstant(declarator.get('init'), seen); + } + } + return value; +} + +function readString(value) { + value = readConstant(value); + if (value.isStringLiteral()) return value.node.value; + return value.isTemplateLiteral() && value.node.expressions.length === 0 ? value.node.quasis[0].value.cooked : undefined; +} + +function actionSourceType(source) { + source = readConstant(source); + if (!source.isObjectExpression()) return undefined; + let type; + for (const property of source.get('properties')) { + // A spread or a computed key could supply or replace the type. + if (!property.isObjectProperty() || property.node.computed) return undefined; + if (keyName(property) === 'type') type = readString(property.get('value')); + } + return type; +} + +/** The physical requirements of a WebDriver action payload, and whether every source in it could be read. */ +function readActionSources(payload) { + payload = readConstant(payload); + if (!payload.isArrayExpression()) return undefined; + const requires = new Set(); + let readable = true; + for (const element of payload.get('elements')) { + const type = element?.node && !element.isSpreadElement() ? actionSourceType(element) : undefined; + if (type === undefined) readable = false; + else if (Object.hasOwn(actionSourceRequirements, type)) requires.add(actionSourceRequirements[type]); + } + return { requires: Object.values(actionSourceRequirements).filter((requirement) => requires.has(requirement)), readable }; +} + +/** A call that sends physical input, with the requirements that input implies. Those are undefined when the call cannot be read. */ +function directInputCall(call) { + const callee = call.get('callee'); + if (!callee.isMemberExpression() && !callee.isOptionalMemberExpression()) return undefined; + const name = memberName(callee); + const argument = call.node.arguments[0] ? call.get('arguments.0') : undefined; + if (valueCommands.has(name)) return { call: name, requires: ['keyboard'] }; + if (name === 'keys' && isBrowser(callee.get('object'))) return { call: 'browser.keys', requires: ['keyboard'] }; + if (name === 'action' && isBrowser(callee.get('object'))) { + const type = argument && readString(argument); + if (type === undefined) return { call: 'browser.action', requires: undefined }; + return Object.hasOwn(actionSourceRequirements, type) + ? { call: 'browser.action', requires: [actionSourceRequirements[type]] } + : undefined; + } + if (name === 'performActions') { + const sources = argument && readActionSources(argument); + if (!sources) return { call: 'performActions', requires: undefined }; + if (sources.requires.length) return { call: 'performActions', requires: sources.requires }; + return sources.readable ? undefined : { call: 'performActions', requires: undefined }; + } + return undefined; +} + +/** + * Rejects a case that also runs in the accessible input lane but sends physical input itself: the keyboard commands, the value + * commands that may type, and WebDriver actions with key, pointer or wheel sources. The check reads literals only. Input sent + * through a helper or an alias is stopped by the accessible lane's host masks when the case runs. + */ +function rejectDirectInput(callback, { name, input, inputSource }) { + if (input === 'physical') return; + let finding; + callback.traverse({ + 'CallExpression|OptionalCallExpression'(call) { + const found = finding ? undefined : directInputCall(call); + if (found) finding = { ...found, path: call }; + }, + }); + if (!finding) return; + const subject = name ? `"${name}"` : 'This case'; + const declared = inputSource === 'declared'; + if (finding.requires === undefined) { + const declaration = declared ? "input: 'physical'" : 'requires for the physical input it sends (keyboard, physical-click or wheel)'; + fail( + finding.path, + `${subject} runs in the accessible input lane, but calls ${finding.call} with an argument this check cannot read. ` + + `Pass a literal list of input sources, or declare ${declaration}.`, + ); + } + const kinds = finding.requires.map((requirement) => requirementInputKinds[requirement]).join(' and '); + const declaration = declared + ? "input: 'physical'" + : `requires: [${finding.requires.map((requirement) => `'${requirement}'`).join(', ')}]`; + fail( + finding.path, + `${subject} runs in the accessible input lane, but ${finding.call} sends physical ${kinds} input. ` + + `Declare ${declaration} so it runs only in the physical lane, or remove the call.`, + ); +} + function compileWdioTest(callback, sourceFile, source) { const file = types.file(types.program([types.expressionStatement(types.cloneNode(callback.node, true))])); const result = transformFromAstSync(file, source, { @@ -365,3 +495,4 @@ module.exports = () => ({ module.exports.collectWdioTests = collectWdioTests; module.exports.compileWdioTest = compileWdioTest; +module.exports.rejectDirectInput = rejectDirectInput; diff --git a/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts b/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts index 67e6ed935c..e620c955b8 100644 --- a/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts +++ b/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts @@ -16,6 +16,10 @@ declare namespace storyTests { function collectWdioTests(program: NodePath): ReadonlyMap; function compileWdioTest(callback: Callback['callback'], sourceFile: string, source: string): string; + function rejectDirectInput( + callback: Callback['callback'], + descriptor: { name?: string; input: 'accessible' | 'physical' | 'both'; inputSource: 'declared' | 'derived' }, + ): void; } export = storyTests; diff --git a/packages/agentic/storybook-desktop/references/commands.md b/packages/agentic/storybook-desktop/references/commands.md index f60208c21c..a82fa228d0 100644 --- a/packages/agentic/storybook-desktop/references/commands.md +++ b/packages/agentic/storybook-desktop/references/commands.md @@ -29,6 +29,8 @@ only). The mode precedence is CLI flag, `DESKTOP_REPORTER`, config, then | `test -- --list` | Discover authored executable story-test IDs | | `test -- --test ` | Run matching isolated story callbacks | | `test -- --input accessible` | Select accessible and `both` descriptor cases | +| `test -- --list --input physical` | Preview a lane's selection and exclusions | +| `driver -- --input accessible` | Start a driver that masks physical input | | `smoke -- --mode stories` | Build/bundle/launch, traverse stories, clean up | | `smoke -- --mode stories-and-tests` | Traverse stories and execute callbacks | @@ -63,10 +65,16 @@ Windows smokes. Storybook test and stories-and-tests smoke commands accept `--input accessible|physical`. The lane filters `desktopIt` cases by their -resolved input and target platform. `driver --input` records the default lane -for subsequent tests against that instance. Legacy callback collections are -not eligible for input-lane runs. Input lanes cannot be combined with an -explicit non-`auto` click mode. +resolved input, target platform, and scope. Without a flag the lane comes from +`DESKTOP_INPUT`, then from the driver instance a `test` attaches to; none of them +means compatibility mode, which runs every case as before. `driver --input` +starts the driver for that lane and records it for subsequent tests against the +instance. An `accessible` driver masks keyboard, physical click, and wheel, and +`test --input accessible` verifies the mask before running anything: a driver +that is not masked fails the run rather than letting a case move the pointer. +Legacy callback collections are not eligible for input-lane runs. Input lanes +cannot be combined with an explicit non-`auto` click mode. See +[story tests](story-tests.md#run-an-input-lane). The TypeScript API exposes the same operations through `DesktopStorybookCli`. Higher-level supervisors should inject their shared reporter, runner, and diff --git a/packages/agentic/storybook-desktop/references/contributing.md b/packages/agentic/storybook-desktop/references/contributing.md index 207be56c30..8a1ac7efea 100644 --- a/packages/agentic/storybook-desktop/references/contributing.md +++ b/packages/agentic/storybook-desktop/references/contributing.md @@ -17,6 +17,16 @@ The channel child in `config/server.cjs` constructs its native driver through helper policy, authority, descriptor publication and settled driver shutdown in that shared harness implementation, not a parallel Storybook copy. +Input lanes cross these areas. The descriptor schema, merge, and per-case +decision belong to `@fluentui-react-native/desktop-driver/authoring`; do not +copy them here. `src/cli/inputLane.ts` resolves the requested lane, +`src/testing/selection.ts` decides every case before launch and renders +`test --list`, `src/testing/laneVerdict.ts` judges a lane run's accounting, and +`config/server.cjs` masks the accessible lane's physical input on the driver. +`config/transform-story-tests.cjs` is both the Metro Babel plugin and the shared +AST library: its `rejectDirectInput` check runs from `src/testing/extractWdioTests.ts` +only, because the Metro transform has no validated descriptor to judge by. + Keep story callbacks statically extractable and authentication checks fail-closed. Update references when commands, supported config, outputs, or readiness contracts change. diff --git a/packages/agentic/storybook-desktop/references/story-tests.md b/packages/agentic/storybook-desktop/references/story-tests.md index 62662edbbd..438c0e09a7 100644 --- a/packages/agentic/storybook-desktop/references/story-tests.md +++ b/packages/agentic/storybook-desktop/references/story-tests.md @@ -7,63 +7,123 @@ owner_paths: # Author story tests -Story tests are statically extracted CSF `wdio` callbacks. Use `desktopIt` -descriptors when declaring input requirements or endpoint scope. Top-level -functions or named callback collections remain supported for compatibility, -but cannot be selected by an input lane. Callbacks must not close over -module-local values. Import Node-only helpers through literal dynamic -`import()` inside the callback. Keep test code out of the native bundle. +Story tests are statically extracted CSF `wdio` callbacks. Declare each case +with `desktopIt`: its descriptor says which input the case needs and where it +runs, so a run can select it for the input lane, platform, and scope it +exercises. Callbacks must not close over module-local values. Import Node-only +helpers through literal dynamic `import()` inside the callback. Keep test code +out of the native bundle. ```tsx import type { StoryObj } from '@storybook/react-native'; +import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; import type { WdioStory } from '@fluentui-react-native/storybook-desktop/testing'; type Story = WdioStory>; export const Default: Story = { - args: { testID: 'save-button' }, - wdio: { - 'is enabled': async ({ browser, expect }) => { - await expect(await browser.$('~save-button')).toBeEnabled(); - }, - 'supports activation': async ({ browser, platform, skip }) => { - const features = browser.capabilities['furn:features']; - if (!features?.physicalClick) { - skip('Physical pointer input is unavailable.'); - return; - } - await (await browser.$('~save-button')).click(); - }, - }, -}; -``` - -For modality-aware tests, import the value helper and use an array of cases: - -```tsx -import { desktopIt, type WdioStory } from '@fluentui-react-native/storybook-desktop/testing'; - -export const Default: WdioStory> = { args: { testID: 'save-button' }, wdio: [ - desktopIt('activates in either input lane', { requires: ['click'] }, async ({ browser, expect }) => { + desktopIt('is enabled', async ({ browser, expect }) => { + await expect(await browser.$('~save-button')).toBeEnabled(); + }), + desktopIt('activates', { requires: ['click'] }, async ({ browser, expect }) => { await (await browser.$('~save-button')).click(); await expect(await browser.$('~saved-message')).toBeDisplayed(); }), - desktopIt('uses a physical pointer', { requires: ['physical-click'], platforms: ['win32'] }, async ({ browser }) => { - await (await browser.$('~save-button')).click(); + desktopIt('activates with Enter', { requires: ['keyboard'], platforms: ['!macos'] }, async ({ browser, expect }) => { + await browser.keys('\uE007'); + await expect(await browser.$('~saved-message')).toBeDisplayed(); }), ], }; ``` -`requires: ['click']` derives to `both` and adapts `element.click()` to the -selected lane: accessibility click in the accessible lane and physical click -in the physical lane. Explicit physical or accessibility requirements derive -their matching lane. `input` may narrow a `both` case but cannot widen a -transport-specific requirement. `platforms` is an allowlist by default; -`['!win32']` excludes that endpoint. Platform selection uses the target -endpoint, so Win32 remains distinct from Windows. +`desktopIt(name, run)` declares a case with no restrictions, so it runs in every +lane on every platform. `desktopIt(name, options, run)` adds a descriptor. The +older object of named callbacks and the single `wdio: async (context) => ...` +form remain supported for compatibility, but a legacy callback has no +descriptor, so an input lane rejects it, listing the story IDs to migrate, +instead of treating it as `both`. + +## Declare how a case runs + +The descriptor is a strict static JSON literal; an unknown field, a computed +value, or an invalid combination fails extraction with source context. + +| Field | Meaning | +| ------------- | ---------------------------------------------------------------------------------------------------- | +| `requires` | Host capabilities the case needs. They also derive the input lane (below). | +| `input` | `accessible`, `physical`, or `both`. Defaults to what `requires` derives, else `both`. | +| `platforms` | Endpoints the case runs on: `macos`, `windows`, `win32`. Omitted means every endpoint. | +| `scope` | Custom scope values, such as `{ 'rn.renderer': ['fabric'] }`. Omitted means every value. | +| `unsupported` | A known gap: `{ platforms, reason, issue? }`. The case is reported unsupported instead of executing. | +| `quarantine` | A temporary gap: `{ issue, reason, expires, platforms?, input? }`, with `expires` as `YYYY-MM-DD`. | + +### Input and requirements + +Most cases need no `input`: their `requires` already says what they send. + +| Requirement | Derived input | Meaning | +| --------------------------------------------------------------------------------------- | ------------- | ---------------------------------------------------------------------- | +| `keyboard`, `physical-click`, `wheel` | `physical` | Operating-system input that moves the real pointer or sends key events | +| `accessibility-click` | `accessible` | The element's accessibility action | +| `click` | `both` | `element.click()`, which follows the lane (below) | +| `accessibility-actions`, `element-screenshot`, `focus`, `screenshot`, `set-window-rect` | `both` | The host must offer the feature | + +A case with no requirements is `both` and runs in every lane. Requiring both a +physical and an accessibility transport is an error; split the case. An explicit +`input` may narrow a `both` case but cannot widen a transport requirement. + +`requires: ['click']` makes `element.click()` follow the lane: an accessibility +click in the accessible lane and a physical click in the physical lane. Without +a lane, an accessible case still uses accessibility clicks and every other case +follows the configured `--click-mode`, so a physical case on a host that masks +physical click is reported unsupported instead of failing its session. Before +a case runs, the host's reported features must meet each requirement. A case it +cannot meet is reported unsupported, never passed, and makes a lane run +incomplete unless the case declares the gap. + +### Platforms and scope + +`platforms` is an allowlist: `['win32']` runs only on Win32. Negate with +`['!win32']` to run everywhere else; a list holds either allowed or negated +endpoints, never both. Platform selection uses the target endpoint, so Win32 +remains distinct from Windows. + +`scope` narrows by values the target holds. Values within a key are alternatives +and keys combine. `rn.renderer` (`fabric` or `paper`) is built in and the +`rn.*` keys are reserved for the framework. A consumer declares its own keys in +the desktop config's `testScopes`, such as `{ flavor: ['stable', 'preview'] }`, +and each target sets its value in `testScope`. A key or value outside the declared +domains, or a key the target does not supply, is a configuration error rather +than a silent exclusion, so a typo cannot drop a case. + +A case that does not match the target is excluded, not skipped: it is not a test +and is counted only in the run's selection totals. + +### Known gaps + +Declare a gap instead of calling `skip` at run time. `unsupported` lists the +endpoints where the behavior does not work yet, with a reason and ideally an +issue. `quarantine` suspends a flaky case until its `expires` date, optionally for +one endpoint or lane. An expired quarantine fails the run before anything +executes. Declared gaps are reported as unsupported with their reason and do not +make a lane run incomplete; a run-time `skip(reason)` does, so reserve it for +conditions no descriptor can state. + +```tsx +desktopIt( + 'closes on Tab and continues past the anchor', + { + requires: ['physical-click', 'keyboard', 'focus'], + unsupported: { platforms: ['win32'], reason: 'Win32 hosts keep the cycle behavior.', issue: '#4377' }, + }, + async ({ browser, expect }) => { + // ... + }, +); +``` The callback receives the real WebdriverIO `browser`, `expect-webdriverio` `expect`, typed `desktop` assertions/story commands, target `platform` (`macos`, @@ -85,10 +145,10 @@ Metro resolves their dependencies. Use native selectors and WebDriver operations; the app has no DOM or JavaScript execution surface. Read native properties through `getProperty()` rather than -ordinary object matchers. Feature-check native actions and explicitly skip -unsupported cases; never turn unsupported state into false. For example, -`getProperty('accessibilityActions')` lists the action names an element -publishes (such as `Toggle` or `Select`) when +ordinary object matchers. Declare the capabilities a case needs in `requires` +rather than feature-checking inside the callback, and never turn unsupported +state into false. For example, `getProperty('accessibilityActions')` lists the +action names an element publishes (such as `Toggle` or `Select`) when `browser.capabilities['furn:features']?.accessibilityActions` is true. Tests in one story run in declaration order, each in its own Node process and @@ -101,12 +161,62 @@ editing callbacks because their digest participates in manifest freshness. filters names. `test` operates against an already running app/driver; smoke owns the complete app lifecycle. +## Run an input lane + +A lane runs only the cases that apply to one way of driving the app. The +`accessible` lane runs `accessible` and `both` cases through accessibility +actions, so it never moves the pointer or sends key events and is safe on a +machine someone is using. The `physical` lane runs `physical` and `both` cases +with real pointer, keyboard, and wheel input, so it needs a machine nobody is +using. Without a lane, every case runs as before (compatibility mode) and click +behavior follows `--click-mode`. + Use `storybook test --input accessible|physical` or -`storybook smoke --mode stories-and-tests --input accessible|physical` to -select declared cases. `storybook driver --input ...` supplies the default -lane to a later `storybook test` invocation. `--input` cannot be combined -with a non-default `--click-mode`. A lane requires descriptor cases and rejects -legacy callback collections rather than silently treating them as `both`. +`storybook smoke --mode stories-and-tests --input accessible|physical` to run +one lane. `DESKTOP_INPUT` supplies the lane when no flag is given. +`storybook driver --input ...` records the default lane for a later +`storybook test` against that driver. `--input` cannot be combined with a +non-default `--click-mode`. `test --list` shows, for each story, the cases the +lane runs with their resolved input, requirements, platforms, and scope, and the +cases it excludes with the reason; add `--input` to preview a lane. + +The accessible lane holds a case to accessible input by masking the host, not by +trusting the case: the driver reports keyboard, physical click, and wheel as +unavailable, so a case that tries them fails instead of silently moving the +pointer. `smoke` and `driver --input accessible` start the driver already +masked. `test --input accessible` attaches to a running driver and verifies the +mask from the session's reported features before it runs anything. A driver that +does not report all three as unavailable stops the run as an infrastructure +failure rather than running unmasked, so attach the lane to a driver started +with `driver --input accessible`. + +A lane that executes no case is not a passing run, and neither is a lane run +that leaves a case unsupported or skipped without a declared gap. Excluded cases +never count against a run: they are not tests. + +## Keep physical input out of accessible cases + +Extraction checks every `desktopIt` case that can run in the accessible lane +(any case that is not `physical`) and fails with a source-context error when its +callback calls an API that sends physical input without declaring it: + +- `browser.keys(...)`, which sends key events (`keyboard`); +- `setValue`, `addValue`, or `clearValue` on any object (`keyboard`); +- `browser.action('key' | 'pointer' | 'wheel')` and `performActions` with `key`, + `pointer`, or `wheel` sources (`keyboard`, `physical-click`, or `wheel`). + +Declare the matching `requires`, or `input: 'physical'` when the case already +declares its input, so it runs only in the physical lane, or remove the call. A +call whose arguments the check cannot read, such as a computed action list, +fails the same way, because it cannot tell what the call sends. The message +names the call and the requirement to declare. + +The check reads the callback only. It does not follow helper functions or aliases, +and `setValue`, `addValue`, and `clearValue` are matched by name, so a method you +define with one of those names is flagged too. Input hidden behind a helper is +instead caught at run time by the accessible lane's host mask. Legacy callbacks +are not checked, and the Metro transform does not run the check; extraction +(`storybook manifest`, `test`, `smoke`) does. Use the component authoring contract for component-specific test assertions. Legacy `parameters.desktopDriver` static plans remain for compatibility and diff --git a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts index c94989b0b2..8729f91a2c 100644 --- a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts +++ b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts @@ -2,7 +2,8 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import type { NativeDriverArtifact } from '@fluentui-react-native/desktop-driver'; +import type { DesktopStoryManifestEntry, NativeDriverArtifact } from '@fluentui-react-native/desktop-driver'; +import { validateDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; import { DesktopServiceScope } from '@fluentui-react-native/desktop-test'; import type { ResolvedDesktopConfig, ResolvedDesktopTarget } from '@fluentui-react-native/desktop-test'; import { selectDesktopRenderer } from '@fluentui-react-native/desktop-test/config'; @@ -11,6 +12,7 @@ import { STORYBOOK_SMOKE_MODE } from '../config/commands'; import { DesktopStorybookConfig, makeDesktopStorybookConfig } from '../config/makeDesktopStorybookConfig'; import { FURN_STORYBOOK_BUNDLE_IDENTIFIER, FURN_STORYBOOK_INSTANCE_ID } from '../config/instance'; import { FURN_STORYBOOK_PLATFORM } from '../config/platforms'; +import { desktopStorybookTargetId } from '../driver/driverManifest'; import type { DesktopCommandRunner, PreparedDesktopCommand, RunningDesktopCommand } from './commandRunner'; import { createDesktopStorybookCommand } from './createDesktopStorybookCommand'; import { @@ -76,6 +78,33 @@ const createEmptyStoryManifest = async (_config: unknown, platform: 'macos' | 'w schemaVersion: 1 as const, }); +function declaredStoryEntry(id: string, cases: readonly (readonly [name: string, options: unknown])[]): DesktopStoryManifestEntry { + return { + id, + name: 'Default', + title: 'Button', + sourcePath: 'button.stories.tsx', + packageName: 'fixture', + tags: [], + wdio: { + exportName: 'Default', + digest: 'declared', + testNames: cases.map(([name]) => name), + cases: cases.map(([name, options]) => ({ name, options: validateDesktopCaseOptions(options) })), + }, + }; +} + +function storyManifestOf(...entries: DesktopStoryManifestEntry[]) { + return async () => ({ + endpoint: 'macos' as const, + schemaVersion: 1 as const, + platformManifestDigest: 'digest', + portablePlanDigest: 'portable', + entries, + }); +} + describe('macOS application leases', () => { test('validates exact running application records and rejects ambiguity', () => { expect( @@ -360,6 +389,115 @@ describe('DesktopStorybookCli', () => { await expect(cli.test('macos', { timeoutMs: 0 })).rejects.toThrow('wdio.timeoutMs'); }); + describe('input lane selection', () => { + const cases = [ + ['adaptive', { requires: ['click'] }], + ['typing', { requires: ['keyboard'] }], + ['win32-only', { platforms: ['win32'] }], + ['paper-only', { scope: { 'rn.renderer': ['paper'] } }], + ['fabric-only', { scope: { 'rn.renderer': ['fabric'] } }], + ] as const; + const config = () => makeDesktopStorybookConfig({ projectRoot: storybookRoot, storyPackages: [] }); + const list = async (flags: string[], ...entries: DesktopStoryManifestEntry[]) => { + const output = { write: jest.fn() }; + await createDesktopStorybookCommand({ config: config(), createStoryManifest: storyManifestOf(...entries), output }).parseAsync([ + 'node', + 'storybook', + 'test', + '--macos', + '--list', + ...flags, + ]); + return JSON.parse(output.write.mock.calls[0][0]); + }; + // An explicit URL and target stand in for a running driver, so selection is what these runs exercise. + const connection = { url: 'https://127.0.0.1:1234', target: 'fixture' }; + const run = (options: Parameters[1], ...entries: DesktopStoryManifestEntry[]) => + new DesktopStorybookCli(config(), { createStoryManifest: storyManifestOf(...entries), output: { write: jest.fn() } }).test('macos', { + ...connection, + ...options, + }); + const legacyEntry = (id: string): DesktopStoryManifestEntry => ({ + ...declaredStoryEntry(id, []), + wdio: { exportName: 'Default', digest: 'legacy' }, + }); + + test.each([ + [ + 'accessible', + ['adaptive', 'fabric-only'], + [ + ['typing', 'input'], + ['win32-only', 'platform'], + ['paper-only', 'scope'], + ], + ], + [ + 'physical', + ['adaptive', 'typing', 'fabric-only'], + [ + ['win32-only', 'platform'], + ['paper-only', 'scope'], + ], + ], + ])('lists what the %s lane runs and why each other matching case is left out', async (input, selected, excluded) => { + const [story] = await list(['--input', input], declaredStoryEntry('button--default', cases)); + expect(story.tests.map(({ name }: { name: string }) => name)).toEqual(selected); + expect(story.excluded.map(({ name, reason }: { name: string; reason: string }) => [name, reason])).toEqual(excluded); + expect(story.tests[0]).toEqual({ name: 'adaptive', input: 'both', inputSource: 'derived', requires: ['click'] }); + }); + + test('lists every declared case that targets the platform when no lane is selected', async () => { + const [story] = await list([], declaredStoryEntry('button--default', cases)); + expect(story.tests.map(({ name }: { name: string }) => name)).toEqual(['adaptive', 'typing', 'fabric-only']); + expect(story.excluded.map(({ reason }: { reason: string }) => reason)).toEqual(['platform', 'scope']); + }); + + test('names every legacy callback a lane cannot run, so they migrate in one pass', async () => { + await expect(run({ input: 'accessible' }, legacyEntry('a--default'), legacyEntry('b--default'))).rejects.toThrow( + 'Input lane "accessible" cannot run legacy wdio callbacks; declare them with desktopIt: a--default, b--default.', + ); + }); + + test('explains an empty selection by what the target and lane excluded', async () => { + await expect(run({ input: 'accessible' }, declaredStoryEntry('button--default', cases.slice(1, 3)))).rejects.toThrow( + 'No executable wdio story tests matched the requested selection (excluded 1 by platform, 0 by scope, 1 by input).', + ); + }); + + test('does not blame a narrow filter for cases it never asked for', async () => { + await expect(run({ input: 'accessible', test: 'typing' }, declaredStoryEntry('button--default', cases))).rejects.toThrow( + 'No executable wdio story tests matched the requested selection (excluded 0 by platform, 0 by scope, 1 by input).', + ); + }); + + test('takes the lane of the running driver instance unless a flag or the environment names one', async () => { + const manifestPath = path.join(storybookRoot, 'storybook-desktop.generated', 'driver-manifest.macos.json'); + const cli = new DesktopStorybookCli(config(), { + createStoryManifest: storyManifestOf(declaredStoryEntry('button--default', cases.slice(1, 2))), + output: { write: jest.fn() }, + }); + fs.mkdirSync(path.dirname(manifestPath), { recursive: true }); + fs.writeFileSync( + manifestPath, + JSON.stringify({ + endpoint: 'macos', + instanceId: cli.instance.id, + renderer: 'fabric', + driverPort: 7100, + targetId: desktopStorybookTargetId(config(), 'macos', 'fabric'), + wdio: { input: 'accessible' }, + }), + ); + try { + // The only case needs a physical keyboard, so only the inherited accessible lane can leave it out. + await expect(cli.test('macos', {})).rejects.toThrow('excluded 0 by platform, 0 by scope, 1 by input'); + } finally { + fs.rmSync(manifestPath, { force: true }); + } + }); + }); + test('starts the config-owned server with platform and connection options', async () => { const runner = new RecordingRunner(); const services = new DesktopServiceScope(runner); @@ -486,6 +624,9 @@ describe('DesktopStorybookCli', () => { expect(fabricManifest.renderer).toBe('fabric'); expect(paperManifest.renderer).toBe('paper'); + // The control process that runs smoke tests has no desktop configuration, so the manifest carries the scope values. + expect(fabricManifest.scope.values).toMatchObject({ 'rn.renderer': 'fabric' }); + expect(paperManifest.scope.values).toMatchObject({ 'rn.renderer': 'paper' }); expect(paperManifest.bridgeNonce).not.toBe(fabricManifest.bridgeNonce); expect(repeatedPaperManifest.bridgeNonce).toBe(paperManifest.bridgeNonce); } finally { @@ -1018,6 +1159,7 @@ describe('DesktopStorybookCli', () => { manifest: await createEmptyStoryManifest(cli.config, 'macos'), platform: 'macos', report: undefined, + scope: { domains: { 'rn.renderer': ['fabric', 'paper'] }, values: { 'rn.renderer': 'fabric' } }, signal: expect.any(AbortSignal), targetId: 'agenticstorybook-macos', }); diff --git a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts index ad62fc01bf..ff7da2de2f 100644 --- a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts +++ b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts @@ -2,8 +2,14 @@ import { spawn } from 'node:child_process'; import fs from 'node:fs'; import { createServer } from 'node:net'; import path from 'node:path'; -import { defaultAgentDescriptor, DesktopServiceScope, resolveDesktopApplication } from '@fluentui-react-native/desktop-test'; +import { + defaultAgentDescriptor, + DesktopServiceScope, + resolveDesktopApplication, + resolveDesktopTestScope, +} from '@fluentui-react-native/desktop-test'; import type { DesktopReportWriter } from '@fluentui-react-native/desktop-test/reporting'; +import { resolveDesktopScopeDomains } from '@fluentui-react-native/desktop-driver/authoring'; import { formatDesktopReporterValue } from '@fluentui-react-native/desktop-driver/cli/reporter'; import type { ResolvedDesktopReporterMode } from '@fluentui-react-native/desktop-driver/cli/reporter'; @@ -38,7 +44,9 @@ import { writeDesktopStoryManifest, } from '../driver/index.js'; import { + desktopStorybookDriverManifestPath, desktopStorybookTargetId, + driverManifestInput, type CreateDesktopStorybookDriverManifestOptions, type DesktopStorybookAgentManifest, type DesktopStorybookDriverManifest, @@ -46,9 +54,11 @@ import { import { NodeDesktopCommandRunner, STORYBOOK_VERBOSE } from './commandRunner.js'; import type { DesktopCommandRunner, PreparedDesktopCommand, RunningDesktopCommand } from './commandRunner.js'; import { formatDesktopStorybookSmokeTestSummary, runDesktopStorybookSmokeTests } from './smokeTests.js'; -import { matchesWdioTest, runWdioStoryTests, selectWdioStories } from '../testing/runWdioTests.js'; +import { runWdioStoryTests } from '../testing/runWdioTests.js'; +import { describeWdioCounts } from '../testing/laneVerdict.js'; +import { decideWdioSelection, describeWdioExclusions, listWdioSelection } from '../testing/selection.js'; +import type { WdioSelectionContext } from '../testing/selection.js'; import { resolveWdioOptions, type DesktopStorybookWdioOptions } from '../config/wdio.js'; -import { matchesDesktopPlatforms } from '@fluentui-react-native/desktop-driver/authoring'; type ResolvedDesktopStorybookInstance = DesktopStorybookInstance & { driverManifestPath?: string; @@ -352,45 +362,20 @@ export class DesktopStorybookCli { ? resolveWdioOptions({ ...initialSettings, input: connection.input }) : initialSettings; const manifest = await this.createStoryManifest(this.config, platform); - const selected = selectWdioStories(manifest, settings, platform); - if (selected.length === 0) { - throw new Error('No executable wdio story tests matched the requested selection.'); + const scope = this.scopeContext(platform); + const selection = decideWdioSelection(manifest, settings, { platform, scope }); + if (selection.stories.length === 0) { + throw new Error(`No executable wdio story tests matched the requested selection${describeWdioExclusions(selection)}.`); } if (options.list) { - const value = selected.map(({ id, sourcePath, wdio }) => ({ - id, - sourcePath, - ...(wdio?.cases - ? { - tests: wdio.cases - .filter( - ({ name, options: descriptor }) => - matchesWdioTest(name, settings.test) && - (!settings.input || descriptor.input === 'both' || descriptor.input === settings.input) && - matchesDesktopPlatforms(descriptor.platforms, platform), - ) - .map(({ name, options: descriptor }) => ({ - name, - input: descriptor.input, - inputSource: descriptor.inputSource, - requires: descriptor.requires ?? [], - platforms: descriptor.platforms, - })), - } - : wdio?.testNames - ? { tests: wdio.testNames.filter((name) => matchesWdioTest(name, settings.test)) } - : {}), - })); + const value = listWdioSelection(selection, settings); if (this.reporterMode === 'json' || this.reporterMode === 'ndjson') this.writeStructured(value); else this.output.write(`${JSON.stringify(value, null, 2)}\n`); return; } const activeConnection = connection ?? this.testConnection(platform, options); if (platform === 'macos' && !options.url) { - await this.writeMacOSApplicationLease( - path.join(this.config.projectRoot, 'storybook-desktop.generated', 'driver-manifest.macos.json'), - settings.timeoutMs, - ); + await this.writeMacOSApplicationLease(desktopStorybookDriverManifestPath(this.config.projectRoot, 'macos'), settings.timeoutMs); } const results = await runWdioStoryTests( { @@ -399,6 +384,7 @@ export class DesktopStorybookCli { config: this.config, manifest, platform, + scope, errorOutput: this.errorOutput, report: this.reporter, signal: this.services.controller.signal, @@ -406,10 +392,14 @@ export class DesktopStorybookCli { }, this.runner, ); - this.output.write( - `Ran ${results.length} executable wdio tests (${results.filter(({ status }) => status === 'passed').length} passed, ` + - `${results.filter(({ status }) => status === 'skipped').length} skipped).\n`, - ); + this.output.write(`Ran ${results.length} executable wdio tests (${describeWdioCounts(results)}).\n`); + } + + /** The scope domains and the values this endpoint's target holds, which declared case `scope` selection reads. */ + private scopeContext(platform: Platforms): NonNullable { + const domains = this.config.desktop?.testScopes ?? resolveDesktopScopeDomains(); + // This run's renderer wins over the host configuration, since `--paper` switches it. + return { domains, values: resolveDesktopTestScope(domains, this.config.getDesktopTarget(platform), this.renderer(platform)) }; } private testConnection( @@ -422,7 +412,7 @@ export class DesktopStorybookCli { } return { targetId: options.target, url: options.url }; } - const manifestPath = path.join(this.config.projectRoot, 'storybook-desktop.generated', `driver-manifest.${platform}.json`); + const manifestPath = desktopStorybookDriverManifestPath(this.config.projectRoot, platform); if (!fs.existsSync(manifestPath)) { throw new Error(`Start storybook driver --${platform} and launch the app before running storybook test.`); } @@ -452,16 +442,7 @@ export class DesktopStorybookCli { `The Storybook driver instance at ${manifestPath} serves an agent; restart storybook driver --${platform} without --agent to run storybook test.`, ); } - const input = - typeof manifest === 'object' && - manifest !== null && - 'wdio' in manifest && - typeof manifest.wdio === 'object' && - manifest.wdio !== null && - 'input' in manifest.wdio && - (manifest.wdio.input === 'accessible' || manifest.wdio.input === 'physical') - ? manifest.wdio.input - : undefined; + const input = driverManifestInput(manifest); return { targetId: manifest.targetId, url: loopbackUrl(manifest.driverPort), ...(input ? { input } : {}) }; } @@ -623,6 +604,7 @@ export class DesktopStorybookCli { driverUrl: loopbackUrl(instance.driverPort), manifest: await this.createStoryManifest(this.config, platform), platform, + scope: this.scopeContext(platform), targetId: this.targetId(platform), report: this.reporter, signal: this.services.controller.signal, @@ -722,7 +704,7 @@ export class DesktopStorybookCli { wdio?: CreateDesktopStorybookDriverManifestOptions['wdio'], ): Promise<{ manifest: DesktopStorybookDriverManifest; path: string }> { const storyManifest = await this.createStoryManifest(this.config, platform); - const outputPath = path.join(this.config.projectRoot, 'storybook-desktop.generated', `driver-manifest.${platform}.json`); + const outputPath = desktopStorybookDriverManifestPath(this.config.projectRoot, platform); const renderer = this.renderer(platform); const driverManifest = createDesktopStorybookDriverManifest({ ...(agent ? { agent } : {}), @@ -732,6 +714,7 @@ export class DesktopStorybookCli { nativeDriver, platform, renderer, + scope: this.scopeContext(platform), storyManifest, ...(wdio ? { wdio } : {}), }); diff --git a/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts b/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts index f7552ff94f..404f6a88d7 100644 --- a/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts +++ b/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts @@ -26,6 +26,7 @@ import type { DesktopStorybookTestOptions, } from './DesktopStorybookCli.js'; import { DesktopStorybookCli } from './DesktopStorybookCli.js'; +import { resolveCommandInputLane, readDriverInstanceInput } from './inputLane.js'; import { loadDesktopStorybookConfig } from './loadConfig.js'; type PlatformFlags = { @@ -276,10 +277,27 @@ export async function runDesktopStorybookCli(argv: readonly string[] = process.a configured: options.config.desktop?.reporting?.events, }); options.reporterMode = reporterMode; + const actionOptions = action.opts(); + const platform = selectedPlatform(actionOptions) ?? options.config.platform; + const { projectRoot } = options.config; + const lane = + action.name() === 'test' || action.name() === 'smoke' + ? resolveCommandInputLane({ + command: action.name() as 'test' | 'smoke', + runsTests: action.name() === 'test' || actionOptions.mode === 'stories-and-tests', + flag: actionOptions.input, + environment: process.env.DESKTOP_INPUT, + driverInput: + action.name() === 'test' && platform && !actionOptions.url && !actionOptions.target + ? () => readDriverInstanceInput(projectRoot, platform) + : undefined, + }) + : undefined; session = await createDesktopReportSession({ ...options.config.desktop?.reporting, artifactsRoot: options.config.desktop?.reporting.artifactsRoot ?? path.join(options.config.projectRoot, 'artifacts', 'desktop'), kind: ['test', 'smoke'].includes(action.name()) ? 'tests' : 'lifecycle', + ...(lane ? { lane } : {}), reporterMode, eventLevel, // A long-lived command's readiness record is its only output; before readiness, a failure reports its summary. diff --git a/packages/agentic/storybook-desktop/src/cli/inputLane.test.ts b/packages/agentic/storybook-desktop/src/cli/inputLane.test.ts new file mode 100644 index 0000000000..1d9202f145 --- /dev/null +++ b/packages/agentic/storybook-desktop/src/cli/inputLane.test.ts @@ -0,0 +1,101 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { readDriverInstanceInput, resolveCommandInputLane } from './inputLane.js'; + +describe('command input lane', () => { + const driverInput = jest.fn((): 'accessible' | 'physical' | undefined => 'physical'); + beforeEach(() => driverInput.mockClear()); + + test('has no lane in compatibility mode', () => { + expect(resolveCommandInputLane({ command: 'test', runsTests: true, driverInput: () => undefined })).toBeUndefined(); + expect(resolveCommandInputLane({ command: 'test', runsTests: true })).toBeUndefined(); + }); + + test('a run that executes no authored tests has no lane to account for', () => { + expect(resolveCommandInputLane({ command: 'smoke', runsTests: false, flag: 'accessible', driverInput })).toBeUndefined(); + expect(driverInput).not.toHaveBeenCalled(); + }); + + test('prefers the flag, then the environment, then the driver instance', () => { + const request = { command: 'test', runsTests: true, driverInput } as const; + expect(resolveCommandInputLane({ ...request, flag: 'accessible', environment: 'physical' })).toMatchObject({ + input: 'accessible', + source: 'option', + }); + expect(resolveCommandInputLane({ ...request, environment: 'accessible' })).toMatchObject({ + input: 'accessible', + source: 'environment', + }); + expect(resolveCommandInputLane(request)).toMatchObject({ input: 'physical', source: 'driver' }); + }); + + test('does not read the driver instance when a flag or environment names the lane', () => { + resolveCommandInputLane({ command: 'test', runsTests: true, flag: 'physical', driverInput }); + resolveCommandInputLane({ command: 'test', runsTests: true, environment: 'physical', driverInput }); + expect(driverInput).not.toHaveBeenCalled(); + }); + + test('labels how an accessible lane is masked by who owns the driver, and never a physical lane', () => { + expect(resolveCommandInputLane({ command: 'smoke', runsTests: true, flag: 'accessible' })).toEqual({ + input: 'accessible', + source: 'option', + masking: 'host-masked', + policyEnforced: false, + }); + expect(resolveCommandInputLane({ command: 'test', runsTests: true, flag: 'accessible' })).toEqual({ + input: 'accessible', + source: 'option', + masking: 'verified-masked', + policyEnforced: false, + }); + expect(resolveCommandInputLane({ command: 'smoke', runsTests: true, flag: 'physical' })).toEqual({ + input: 'physical', + source: 'option', + policyEnforced: false, + }); + }); + + test.each([ + ['flag', { flag: 'both' }, '--input must be accessible or physical.'], + ['environment', { environment: 'Physical' }, 'DESKTOP_INPUT must be accessible or physical.'], + ['empty flag', { flag: '' }, '--input must be accessible or physical.'], + ['empty environment', { environment: '' }, 'DESKTOP_INPUT must be accessible or physical.'], + ] as const)('rejects an invalid %s', (_name, request, message) => { + expect(() => resolveCommandInputLane({ command: 'test', runsTests: true, ...request })).toThrow( + expect.objectContaining({ message, code: 'config' }), + ); + }); +}); + +describe('driver instance input', () => { + let root: string; + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'storybook-input-lane-')); + fs.mkdirSync(path.join(root, 'storybook-desktop.generated')); + }); + afterEach(() => fs.rmSync(root, { recursive: true, force: true })); + const write = (contents: string) => + fs.writeFileSync(path.join(root, 'storybook-desktop.generated', 'driver-manifest.win32.json'), contents); + + test('reads the lane a driver instance was started for', () => { + write(JSON.stringify({ wdio: { input: 'accessible' } })); + expect(readDriverInstanceInput(root, 'win32')).toBe('accessible'); + }); + + test.each([ + ['no lane', JSON.stringify({ wdio: {} })], + ['an unknown lane', JSON.stringify({ wdio: { input: 'both' } })], + ['no wdio settings', JSON.stringify({})], + ['malformed JSON', '{'], + ['a non-object manifest', 'null'], + ])('declares none for %s', (_name, contents) => { + write(contents); + expect(readDriverInstanceInput(root, 'win32')).toBeUndefined(); + }); + + test('declares none without a manifest for the endpoint', () => { + expect(readDriverInstanceInput(root, 'macos')).toBeUndefined(); + }); +}); diff --git a/packages/agentic/storybook-desktop/src/cli/inputLane.ts b/packages/agentic/storybook-desktop/src/cli/inputLane.ts new file mode 100644 index 0000000000..8a6e36dd39 --- /dev/null +++ b/packages/agentic/storybook-desktop/src/cli/inputLane.ts @@ -0,0 +1,42 @@ +import fs from 'node:fs'; + +import type { DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; +import { createDesktopInputLane, requestedDesktopInputLane } from '@fluentui-react-native/desktop-test/reporting'; +import type { DesktopInputLaneReport } from '@fluentui-react-native/desktop-test/reporting'; + +import type { Platforms } from '../config/platforms.js'; +import { desktopStorybookDriverManifestPath, driverManifestInput } from '../driver/driverManifest.js'; + +export type CommandInputLaneRequest = { + command: 'test' | 'smoke'; + /** `smoke` runs authored tests only in its stories-and-tests mode; a run without tests has no lane to account for. */ + runsTests: boolean; + flag?: string; + environment?: string; + /** The lane the attached driver instance was started for. Only `test` attaches to one, and only without `--url`. */ + driverInput?: () => DesktopInputLane | undefined; +}; + +/** + * The lane a `test` or `smoke` run holds its tests to, and where it came from: `--input`, then `DESKTOP_INPUT`, then + * the driver instance `test` attaches to. No lane is compatibility mode, which keeps the behavior from before lanes. + * An accessible lane's masking depends on who owns the driver: `smoke` starts it masked, while `test` attaches to one + * and verifies the mask before it runs anything. + */ +export function resolveCommandInputLane(request: CommandInputLaneRequest): DesktopInputLaneReport | undefined { + if (!request.runsTests) return undefined; + const masking = request.command === 'smoke' ? 'host-masked' : 'verified-masked'; + const requested = requestedDesktopInputLane({ flag: request.flag, environment: request.environment }); + if (requested) return createDesktopInputLane(requested.input, requested.source, masking); + const input = request.driverInput?.(); + return input ? createDesktopInputLane(input, 'driver', masking) : undefined; +} + +/** The lane the running driver instance for an endpoint declares. A missing or unreadable manifest declares none. */ +export function readDriverInstanceInput(projectRoot: string, platform: Platforms): DesktopInputLane | undefined { + try { + return driverManifestInput(JSON.parse(fs.readFileSync(desktopStorybookDriverManifestPath(projectRoot, platform), 'utf8'))); + } catch { + return undefined; + } +} diff --git a/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts b/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts index da8b2d0bac..c07b9ee7b2 100644 --- a/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts +++ b/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts @@ -4,10 +4,13 @@ import path from 'node:path'; import type { DesktopStoryManifest, DesktopStoryManifestEntry } from '@fluentui-react-native/desktop-driver'; import type { DesktopStoryRunResult } from '@fluentui-react-native/desktop-driver/authoring'; +import { validateDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopReportWriter } from '@fluentui-react-native/desktop-test/reporting'; import { makeDesktopStorybookConfig } from '../config/makeDesktopStorybookConfig.js'; import type { runWdioStoryTests } from '../testing/runWdioTests.js'; import { WdioStoryTestRunError } from '../testing/runWdioTests.js'; +import { csfTestId } from '../testing/selection.js'; import { formatDesktopStorybookSmokeTestSummary, runDesktopStorybookSmokeTests, @@ -31,6 +34,32 @@ function story(id: string, wdio = false): DesktopStoryManifestEntry { }; } +function declaredStory(id: string, cases: readonly (readonly [name: string, options: unknown])[]): DesktopStoryManifestEntry { + return { + ...story(id), + tests: undefined, + wdio: { + exportName: 'Default', + digest: 'callback', + testNames: cases.map(([name]) => name), + cases: cases.map(([name, descriptor]) => ({ name, options: validateDesktopCaseOptions(descriptor) })), + }, + }; +} + +function fakeReport(root: string) { + const emit = jest.fn(); + const report = { + runId: 'run', + root, + emit, + artifactPath: (name: string) => path.join(root, name), + failure: jest.fn(), + fail: jest.fn(), + } as unknown as DesktopReportWriter; + return { report, emit }; +} + const manifest: DesktopStoryManifest = { endpoint: 'windows', entries: [story(buttonId)], @@ -290,4 +319,99 @@ describe('runDesktopStorybookSmokeTests', () => { await expect(runDesktopStorybookSmokeTests(options, connect)).rejects.toThrow('No desktop story tests'); expect(connect).not.toHaveBeenCalled(); }); + + describe('input lanes', () => { + test('announces every selected case up front and runs each story against that announcement', async () => { + const { report, emit } = fakeReport(root); + options.report = report; + options.input = 'accessible'; + options.scope = { domains: { 'rn.renderer': ['fabric', 'paper'] }, values: { 'rn.renderer': 'fabric' } }; + options.manifest = { + ...manifest, + entries: [ + declaredStory(buttonId, [ + ['plain', {}], + ['typed', { requires: ['keyboard'] }], + ]), + declaredStory(inputId, [['typing', { requires: ['accessibility-click'] }]]), + ], + }; + const runWdio: jest.MockedFunction = jest.fn(async ({ story }) => [ + { storyId: story!, testName: 'ran', status: 'passed' as const, durationMs: 1 }, + ]); + + await runDesktopStorybookSmokeTests(options, jest.fn(), runWdio); + + const events = emit.mock.calls.map(([event]) => event); + expect(events.filter(({ kind }) => kind === 'selection')).toEqual([ + { kind: 'selection', payload: { excluded: { platform: 0, scope: 0, input: 1 } } }, + ]); + expect(events.filter(({ kind }) => kind === 'work-selected').map(({ payload }) => payload.workId)).toEqual([ + `smoke:${buttonId}`, + `smoke:${inputId}`, + csfTestId(buttonId, 'plain'), + csfTestId(inputId, 'typing'), + ]); + expect(runWdio).toHaveBeenCalledWith( + expect.objectContaining({ story: buttonId, input: 'accessible', scope: options.scope, announced: true, optional: true }), + undefined, + ); + }); + + test('rejects legacy callbacks before connecting', async () => { + options.input = 'physical'; + options.manifest = { ...manifest, entries: [{ ...story(buttonId, true), tests: undefined }] }; + const connect = jest.fn(); + await expect(runDesktopStorybookSmokeTests(options, connect)).rejects.toThrow( + `Input lane "physical" cannot run legacy wdio callbacks; declare them with desktopIt: ${buttonId}.`, + ); + expect(connect).not.toHaveBeenCalled(); + }); + + test('states a lane selection that left every case out and explains why', async () => { + const { report, emit } = fakeReport(root); + options.report = report; + options.input = 'accessible'; + options.manifest = { ...manifest, entries: [declaredStory(buttonId, [['typed', { requires: ['keyboard'] }]])] }; + await expect(runDesktopStorybookSmokeTests(options, jest.fn())).rejects.toThrow( + 'No desktop story tests matched the smoke selection (excluded 0 by platform, 0 by scope, 1 by input).', + ); + expect(emit).toHaveBeenCalledWith({ kind: 'selection', payload: { excluded: { platform: 0, scope: 0, input: 1 } } }); + }); + + test('a lane that executed no wdio test is not a passing run even when static plans passed', async () => { + options.input = 'accessible'; + options.manifest = { ...manifest, entries: [{ ...story(buttonId), wdio: declaredStory(buttonId, [['plain', {}]]).wdio }] }; + const connect: DesktopStorybookSmokeConnector = async () => ({ + delete: async () => undefined, + listStories: async () => options.manifest, + runStoryTests: async () => planResult(), + }); + const runWdio = jest.fn(async () => [ + { storyId: buttonId, testName: 'plain', status: 'unsupported' as const, gap: 'unsupported' as const, durationMs: 0 }, + ]); + await expect(runDesktopStorybookSmokeTests(options, connect, runWdio)).rejects.toThrow( + 'Input lane "accessible" executed no wdio story tests; it is not a passing run.', + ); + }); + }); + + test('does not pass a cohort that only skipped or was unsupported', async () => { + options.manifest = { ...manifest, entries: [{ ...story(buttonId, true), tests: undefined }] }; + const runWdio = jest.fn(async () => [ + { storyId: buttonId, status: 'unsupported' as const, gap: 'unsupported' as const, durationMs: 0 }, + ]); + await expect(runDesktopStorybookSmokeTests(options, jest.fn(), runWdio)).rejects.toThrow('only skipped or unsupported'); + }); + + test('breaks unsupported down in the summary only when something is unsupported', () => { + const wdio = [{ storyId: buttonId, status: 'passed' as const, durationMs: 1 }]; + expect(formatDesktopStorybookSmokeTestSummary({ ...planResult(), wdio })).toBe('Ran 2 desktop story tests (2 passed, 0 skipped).'); + expect( + formatDesktopStorybookSmokeTestSummary({ + ...planResult(), + wdio: [...wdio, { storyId: inputId, status: 'unsupported' as const, gap: 'unsupported' as const, durationMs: 0 }], + }), + ).toBe('Ran 3 desktop story tests (2 passed, 0 skipped, 1 unsupported (1 declared, 0 quarantined)).'); + }); }); diff --git a/packages/agentic/storybook-desktop/src/cli/smokeTests.ts b/packages/agentic/storybook-desktop/src/cli/smokeTests.ts index afa53f720f..c45a586c70 100644 --- a/packages/agentic/storybook-desktop/src/cli/smokeTests.ts +++ b/packages/agentic/storybook-desktop/src/cli/smokeTests.ts @@ -12,7 +12,9 @@ import type { DesktopWebdriverOptions, DesktopWebdriverSession } from '@fluentui import type { DesktopStorybookConfig } from '../config/makeDesktopStorybookConfig.js'; import type { Platforms } from '../config/platforms.js'; import { resolveWdioOptions } from '../config/wdio.js'; -import { runWdioStoryTests, selectWdioStories, WdioStoryTestRunError, type WdioStoryTestResult } from '../testing/runWdioTests.js'; +import { describeWdioCounts, hasExecutedWdioCase } from '../testing/laneVerdict.js'; +import { runWdioStoryTests, WdioStoryTestRunError, type WdioStoryTestResult } from '../testing/runWdioTests.js'; +import { decideWdioSelection, describeWdioExclusions, plannedWdioWork, type WdioSelectionContext } from '../testing/selection.js'; import { writeDesktopStorybookFailure, type DesktopStorybookErrorOutput } from '../../config/diagnostics.cjs'; import type { DesktopCommandRunner } from './commandRunner.js'; @@ -22,6 +24,8 @@ export type DesktopStorybookSmokeConnector = (options: DesktopWebdriverOptions) export type DesktopStorybookSmokeTestOptions = { input?: 'accessible' | 'physical'; + /** The scope domains and values of this target, for case `scope` selection. */ + scope?: WdioSelectionContext['scope']; artifactsRoot?: string; config: DesktopStorybookConfig; driverUrl: string; @@ -60,10 +64,16 @@ export async function runDesktopStorybookSmokeTests( const planStoryIds = new Set( selectDesktopStoryTests(manifest, platform, { story: storyPattern, tag: testTag ?? 'desktop-e2e' }).map(({ entry }) => entry.id), ); - const wdioStoryIds = new Set(selectWdioStories(manifest, wdioSettings, platform).map(({ id }) => id)); + const wdioSelection = decideWdioSelection(manifest, wdioSettings, { platform, scope: options.scope }); + const wdioStoryIds = new Set(wdioSelection.stories.map(({ id }) => id)); + const { platform: byPlatform, scope: byScope, input: byInput } = wdioSelection.excluded; + // A lane always states what it excluded, even when nothing; compatibility mode only when something was excluded. + if (options.report && (wdioSettings.input || byPlatform + byScope + byInput > 0)) { + options.report.emit({ kind: 'selection', payload: { excluded: wdioSelection.excluded } }); + } const storyIds = [...new Set([...planStoryIds, ...wdioStoryIds])].sort(); if (storyIds.length === 0) { - throw new Error('No desktop story tests matched the smoke selection.'); + throw new Error(`No desktop story tests matched the smoke selection${describeWdioExclusions(wdioSelection)}.`); } const artifacts = new ArtifactManager( options.artifactsRoot ?? @@ -77,6 +87,8 @@ export async function runDesktopStorybookSmokeTests( const failures: unknown[] = []; const staticFailures = new Map(); for (const storyId of storyIds) options.report?.emit({ kind: 'work-selected', payload: { workId: `smoke:${storyId}`, label: storyId } }); + // Every selected case is announced up front, so a fail-fast stop leaves the remaining cases accounted for as not run. + for (const payload of plannedWdioWork(wdioSelection, wdioSettings)) options.report?.emit({ kind: 'work-selected', payload }); for (const storyId of storyIds) { const scope = `smoke:${storyId}`; @@ -106,6 +118,7 @@ export async function runDesktopStorybookSmokeTests( config, manifest, platform, + scope: options.scope, story: storyId, targetId: options.targetId, url: options.driverUrl, @@ -113,6 +126,7 @@ export async function runDesktopStorybookSmokeTests( report: options.report, signal: options.signal, optional: true, + announced: true, }, options.commandRunner, )), @@ -154,9 +168,18 @@ export async function runDesktopStorybookSmokeTests( } failures.push(new Error('Executable smoke tests reported failures.')); } - const allSkipped = tests.length + wdio.length > 0 && [...tests, ...wdio].every(({ status }) => status === 'skipped'); + const allSkipped = + tests.length + wdio.length > 0 && [...tests, ...wdio].every(({ status }) => status === 'skipped' || status === 'unsupported'); if (allSkipped && failures.length === 0) - failures.push(new Error('Required desktop smoke cohort only skipped tests; it is not a passing run.')); + failures.push(new Error('Required desktop smoke cohort only skipped or unsupported tests; it is not a passing run.')); + // Smoke runs one story at a time as optional, so the lane's requirement to execute something is judged for the cohort here. + if (wdioSettings.input && failures.length === 0 && !hasExecutedWdioCase(wdio)) { + failures.push( + new Error( + `Input lane "${wdioSettings.input}" executed no wdio story tests${describeWdioExclusions(wdioSelection)}; it is not a passing run.`, + ), + ); + } const result: DesktopStorybookSmokeTestResult = { endpoint: platform, finishedAt: new Date().toISOString(), @@ -289,9 +312,7 @@ async function runStoryPlans( export function formatDesktopStorybookSmokeTestSummary(result: DesktopStorybookSmokeTestResult): string { const results = [...result.tests, ...(result.wdio ?? [])]; - const passed = results.filter(({ status }) => status === 'passed').length; - const skipped = results.filter(({ status }) => status === 'skipped').length; - return `Ran ${results.length} desktop story tests (${passed} passed, ${skipped} skipped).`; + return `Ran ${results.length} desktop story tests (${describeWdioCounts(results)}).`; } function errorMessage(error: unknown): string { diff --git a/packages/agentic/storybook-desktop/src/config/integration.ts b/packages/agentic/storybook-desktop/src/config/integration.ts index 3d1b8f00c3..6bf1071cde 100644 --- a/packages/agentic/storybook-desktop/src/config/integration.ts +++ b/packages/agentic/storybook-desktop/src/config/integration.ts @@ -32,7 +32,7 @@ export function storybook(options: DesktopStorybookConfigOptions = {}): Storyboo configure(projectRoot, desktop) { return new DesktopStorybookConfig({ ...options, projectRoot }, desktop); }, - async run({ config, target, supervisor, action, agent, reporterMode, onReady }) { + async run({ config, target, supervisor, action, agent, input, reporterMode, onReady }) { const { DesktopStorybookCli } = await import('@fluentui-react-native/storybook-desktop/cli'); const api = new DesktopStorybookCli(this.configure(config.projectRoot, config), { paper: target.endpoint === 'macos' && target.host.renderer === 'paper', @@ -43,7 +43,7 @@ export function storybook(options: DesktopStorybookConfigOptions = {}): Storyboo ...(onReady ? { onReady } : {}), }); if (action === 'serve') await api.driver(target.endpoint, agent ? { agent } : {}); - else if (action === 'test') await api.smoke(target.endpoint, { mode: 'stories-and-tests' }); + else if (action === 'test') await api.smoke(target.endpoint, { mode: 'stories-and-tests', ...(input ? { input } : {}) }); else await api[action](target.endpoint); }, }; diff --git a/packages/agentic/storybook-desktop/src/config/makeDesktopStorybookConfig.ts b/packages/agentic/storybook-desktop/src/config/makeDesktopStorybookConfig.ts index f65a554091..b61e6f19e6 100644 --- a/packages/agentic/storybook-desktop/src/config/makeDesktopStorybookConfig.ts +++ b/packages/agentic/storybook-desktop/src/config/makeDesktopStorybookConfig.ts @@ -116,8 +116,11 @@ export type DesktopStorybookConfigOptions = StorySettings & { */ platformOptions?: DesktopPlatformOptionsMap; - /** Defaults for executable story callbacks, run by `storybook-desktop test`. */ - wdio?: DesktopStorybookWdioOptions; + /** + * Defaults for executable story callbacks, run by `storybook-desktop test`. The input lane is chosen per run with + * `--input` or `DESKTOP_INPUT`, so it cannot be configured here. + */ + wdio?: Omit; }; export type ResolvedPackage = { @@ -532,6 +535,10 @@ function defaultSmokeOptions(config: DesktopStorybookConfig, platform: Platforms } function normalizeConfig(config: DesktopStorybookConfigOptions, projectRoot: string): DesktopStorybookConfigOptions { + // A configured lane would run tests the report session never holds to a lane, so the lane has one per-run source. + if (config.wdio && 'input' in config.wdio) { + throw new TypeError('wdio.input is chosen per run with --input or DESKTOP_INPUT; it cannot be configured.'); + } const wdio = resolveWdioOptions(config.wdio); validatePlatforms(config.platforms, 'config'); validatePlatformSettings(config.platformSettings, 'config'); diff --git a/packages/agentic/storybook-desktop/src/config/wdio.test.ts b/packages/agentic/storybook-desktop/src/config/wdio.test.ts index 996fdf90c4..e275ae5b52 100644 --- a/packages/agentic/storybook-desktop/src/config/wdio.test.ts +++ b/packages/agentic/storybook-desktop/src/config/wdio.test.ts @@ -25,6 +25,11 @@ describe('Storybook-owned wdio configuration', () => { expect(() => resolveWdioOptions({ tag: ' ' })).toThrow('wdio.tag'); }); + test('rejects a configured input lane, which is chosen per run', () => { + // @ts-expect-error `input` is not a configurable default. + expect(() => makeDesktopStorybookConfig({ wdio: { input: 'accessible' } })).toThrow('wdio.input is chosen per run'); + }); + test('resolves an explicit input lane and rejects conflicting click configuration', () => { const previous = process.env.DESKTOP_INPUT; delete process.env.DESKTOP_INPUT; diff --git a/packages/agentic/storybook-desktop/src/driver/driverManifest.ts b/packages/agentic/storybook-desktop/src/driver/driverManifest.ts index 69dd15764d..3e4ac5b7a0 100644 --- a/packages/agentic/storybook-desktop/src/driver/driverManifest.ts +++ b/packages/agentic/storybook-desktop/src/driver/driverManifest.ts @@ -3,12 +3,14 @@ import fs from 'node:fs'; import path from 'node:path'; import type { DesktopStoryManifest, NativeDesktopApplicationDescriptor, NativeDriverArtifact } from '@fluentui-react-native/desktop-driver'; +import type { DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; import { selectDesktopRenderer } from '@fluentui-react-native/desktop-test/config'; import type { DesktopStorybookConfig } from '../config/makeDesktopStorybookConfig.js'; import type { DesktopStorybookInstance } from '../config/instance.js'; import type { Platforms } from '../config/platforms.js'; import type { DesktopStorybookWdioOptions } from '../config/wdio.js'; +import type { WdioSelectionContext } from '../testing/selection.js'; /** Asks the channel service to protect its driver with a grant authority and publish one agent's connection. */ export type DesktopStorybookAgentManifest = { @@ -43,6 +45,11 @@ export type DesktopStorybookDriverManifest = { portablePlanDigest: string; renderer: 'fabric' | 'paper'; schemaVersion: 2; + /** + * The scope domains and the values this target holds, which declared test `scope` selection reads. The driver's + * CLI resolves them from the full desktop configuration, which a smoke control process does not have. + */ + scope?: NonNullable; storyManifest: DesktopStoryManifest; storybookPort: number; targetId: string; @@ -58,6 +65,7 @@ export type CreateDesktopStorybookDriverManifestOptions = { nativeDriver: NativeDriverArtifact; platform: Platforms; renderer?: DesktopStorybookDriverManifest['renderer']; + scope?: DesktopStorybookDriverManifest['scope']; storyManifest: DesktopStoryManifest; wdio?: DesktopStorybookWdioOptions; }; @@ -81,6 +89,7 @@ export function createDesktopStorybookDriverManifest({ nativeDriver, platform, renderer = platform === 'win32' ? 'paper' : 'fabric', + scope, storyManifest, wdio, }: CreateDesktopStorybookDriverManifestOptions): DesktopStorybookDriverManifest { @@ -119,6 +128,9 @@ export function createDesktopStorybookDriverManifest({ portablePlanDigest: storyManifest.portablePlanDigest, renderer, schemaVersion: 2, + ...(scope + ? { scope: Object.freeze({ domains: Object.freeze({ ...scope.domains }), values: Object.freeze({ ...scope.values }) }) } + : {}), storyManifest, storybookPort: instance.storybookPort, targetId: desktopStorybookTargetId(config, platform, renderer), @@ -136,3 +148,15 @@ export function writeDesktopStorybookDriverManifest(manifest: DesktopStorybookDr fs.writeFileSync(outputPath, content); } } + +/** Where the running driver instance for an endpoint publishes its manifest. */ +export function desktopStorybookDriverManifestPath(projectRoot: string, platform: Platforms): string { + return path.join(projectRoot, 'storybook-desktop.generated', `driver-manifest.${platform}.json`); +} + +/** The input lane a driver instance was started for, read from any parsed manifest; absent when it declares none. */ +export function driverManifestInput(manifest: unknown): DesktopInputLane | undefined { + const wdio = typeof manifest === 'object' && manifest !== null && 'wdio' in manifest ? manifest.wdio : undefined; + const input = typeof wdio === 'object' && wdio !== null && 'input' in wdio ? wdio.input : undefined; + return input === 'accessible' || input === 'physical' ? input : undefined; +} diff --git a/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts b/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts index bdea026b70..c7bf96822d 100644 --- a/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts +++ b/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts @@ -94,6 +94,64 @@ describe('desktop Storybook server integration', () => { expect(output.join('')).toContain('WebDriver '); }); + test.each([ + ['accessible', { keyboard: false, physicalClick: false, wheel: false }], + ['physical', { keyboard: true, physicalClick: true, wheel: true }], + ] as const)('starts a driver for the %s input lane with matching physical input features', async (input, expected) => { + const [storybookPort, driverPort] = await Promise.all([getAvailablePort(), getAvailablePort()]); + const temporaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-storybook-driver-')); + const manifestPath = path.join(temporaryDirectory, 'driver-manifest.json'); + const projectRoot = path.resolve(__dirname, '../../../../../apps/storybook'); + fs.writeFileSync( + manifestPath, + JSON.stringify({ ...createIntegrationManifest(temporaryDirectory, storybookPort, driverPort), wdio: { input } }), + ); + const sink = { write: () => true }; + const reporter = new DesktopRunReporter({ + artifactsRoot: temporaryDirectory, + kind: 'lifecycle', + output: sink, + errorOutput: sink, + summary: false, + }); + const transport = await startReportTransport(reporter); + const runner = new NodeDesktopCommandRunner({ reporter, environment: transport.environment }); + const child = runner.start({ + command: process.execPath, + args: [path.resolve(__dirname, '../../config/server-runner.cjs')], + cwd: projectRoot, + env: { + STORYBOOK_CONFIG_PATH: path.join(projectRoot, 'src'), + STORYBOOK_DRIVER_MANIFEST: manifestPath, + STORYBOOK_NATIVE_DRIVER_FAKE: '1', + STORYBOOK_PROJECT_ROOT: projectRoot, + STORYBOOK_SMOKE_MODE: 'stories-and-tests', + STORYBOOK_WS_PORT: String(storybookPort), + }, + }); + + try { + await waitForResponse(loopbackUrl(driverPort, '/status')); + const client = createDesktopDriverClient({ url: loopbackUrl(driverPort, '') }); + const session = await client.newSession({ + alwaysMatch: { platformName: 'windows', 'furn:target': 'integration-windows', 'furn:launchMode': 'attach' }, + }); + // The accessible lane trusts this report, so the driver itself must show physical input disabled. + expect(session.capabilities['furn:features']).toMatchObject(expected); + await session.delete(); + } finally { + try { + await child.stop(); + await child.completed; + await runner.close(); + await transport.close(); + await reporter.finish(); + } finally { + fs.rmSync(temporaryDirectory, { force: true, recursive: true }); + } + } + }); + test('refuses an agent connection from the fake driver', async () => { const [storybookPort, driverPort] = await Promise.all([getAvailablePort(), getAvailablePort()]); const temporaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-storybook-driver-')); diff --git a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts index 0a5ff18bb4..16766657bb 100644 --- a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts +++ b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts @@ -285,4 +285,115 @@ describe('extractWdioTests', () => { expect(digest(source.replace("'test'", "'changed'"))).not.toBe(digest(source)); expect(digest(source.replace('console.log', '/* source change */ console.log'))).not.toBe(digest(source)); }); + + describe('direct physical input', () => { + const desktopCase = (body: string, options?: string) => + `import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; + export const Example = { wdio: [desktopIt('case', ${options === undefined ? '' : `${options}, `}async (context) => { const { browser } = context; ${body} })] };`; + const key = `{ type: 'key', id: 'keyboard', actions: [{ type: 'keyDown', value: 'a' }] }`; + const pointer = `{ type: 'pointer', id: 'mouse', actions: [{ type: 'pointerMove', x: 1, y: 1 }] }`; + const wheel = `{ type: 'wheel', id: 'wheel', actions: [{ type: 'scroll', deltaY: 10 }] }`; + const pause = `{ type: 'none', id: 'pause', actions: [{ type: 'pause', duration: 100 }] }`; + + test.each([ + ['browser.keys', `await browser.keys('a');`, /browser\.keys sends physical keyboard input.*requires: \['keyboard'\]/], + ['an optional browser.keys', `await browser?.keys('a');`, /browser\.keys sends physical keyboard input/], + ['a computed browser.keys', `await browser['keys']('a');`, /browser\.keys sends physical keyboard input/], + ['a keys call through the context', `await context.browser.keys('a');`, /browser\.keys sends physical keyboard input/], + ['setValue', `await (await browser.$('~input')).setValue('a');`, /setValue sends physical keyboard input/], + ['addValue', `await (await browser.$('~input')).addValue('a');`, /addValue sends physical keyboard input/], + ['clearValue', `await (await browser.$('~input')).clearValue();`, /clearValue sends physical keyboard input/], + ['key actions', `await browser.performActions([${key}]);`, /performActions sends physical keyboard input/], + ['pointer actions', `await browser.performActions([${pointer}]);`, /sends physical pointer input.*requires: \['physical-click'\]/], + ['wheel actions', `await browser.performActions([${wheel}]);`, /sends physical wheel input.*requires: \['wheel'\]/], + [ + 'actions with several physical sources', + `await browser.performActions([${wheel}, ${pause}, ${key}]);`, + /sends physical keyboard and wheel input.*requires: \['keyboard', 'wheel'\]/, + ], + [ + 'actions held in a constant', + `const actions = [${pointer}]; await browser.performActions(actions);`, + /sends physical pointer input/, + ], + ['an action builder', `await browser.action('key').down('a').perform();`, /browser\.action sends physical keyboard input/], + [ + 'a call inside a nested function', + `await browser.waitUntil(async () => { await browser.keys('a'); return true; });`, + /browser\.keys sends physical keyboard input/, + ], + [ + 'actions built elsewhere', + `const buildActions = () => []; await browser.performActions(buildActions());`, + /performActions with an argument this check cannot read/, + ], + ['actions held in a variable that changes', `let actions = []; actions = [1]; await browser.performActions(actions);`, /cannot read/], + ['actions with a spread source', `const sources = []; await browser.performActions([...sources]);`, /cannot read/], + [ + 'actions with a source that has a spread', + `const source = {}; await browser.performActions([{ ...source, id: 'a' }]);`, + /cannot read/, + ], + ['actions without an argument', `await browser.performActions();`, /cannot read/], + [ + 'an action builder with a variable type', + `await browser.action(context.type).perform();`, + /browser\.action with an argument this check cannot read/, + ], + ])('rejects %s in a case that also runs in the accessible lane', (_name, body, message) => { + expect(() => extractWdioTests(desktopCase(body), sourceFile)).toThrow(message); + }); + + test.each([ + ['an explicit both', `{ input: 'both' }`], + ['an explicit accessible case', `{ input: 'accessible' }`], + ['an accessibility click', `{ requires: ['accessibility-click'] }`], + ['neutral requirements', `{ requires: ['click', 'focus'] }`], + ])('rejects the keyboard in %s with the case name and source location', (_name, options) => { + let error: Error | undefined; + try { + extractWdioTests(desktopCase(`await browser.keys('a');`, options), sourceFile); + } catch (caught) { + error = caught as Error; + } + expect(error?.message).toContain('"case" runs in the accessible input lane'); + expect(error?.message).toContain(sourceFile); + }); + + test('suggests the declaration that matches how the case chose its input', () => { + expect(() => extractWdioTests(desktopCase(`await browser.keys('a');`, `{ input: 'both' }`), sourceFile)).toThrow( + /Declare input: 'physical'/, + ); + expect(() => extractWdioTests(desktopCase(`await browser.keys('a');`), sourceFile)).toThrow(/Declare requires: \['keyboard'\]/); + }); + + test.each([ + ['derives physical from keyboard', `{ requires: ['keyboard'] }`], + ['derives physical from a physical click', `{ requires: ['physical-click', 'wheel'] }`], + ['declares physical', `{ input: 'physical' }`], + ])('allows direct input in a case that %s', (_name, options) => { + const body = `await browser.keys('a'); await (await browser.$('~input')).setValue('a'); await browser.performActions([${key}, ${pointer}, ${wheel}]);`; + expect(() => extractWdioTests(desktopCase(body, options), sourceFile)).not.toThrow(); + }); + + test('allows calls that do not send physical input', () => { + const body = ` + const names = Object.keys({ a: 1 }); + const map = new Map([[1, 2]]); + for (const entry of map.keys()) names.push(String(entry)); + await browser.performActions([${pause}]); + await browser.performActions([]); + const store = { setValueLater: () => {} }; + store.setValueLater(); + await (await browser.$('~button')).click(); + `; + expect(() => extractWdioTests(desktopCase(body), sourceFile)).not.toThrow(); + }); + + test('does not read legacy callbacks, which declare no input', () => { + expect(() => + extractWdioTests(`export const Example = { wdio: { legacy: async ({ browser }) => { await browser.keys('a'); } } };`, sourceFile), + ).not.toThrow(); + }); + }); }); diff --git a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts index 58e2eaf30f..4bf2066810 100644 --- a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts +++ b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts @@ -6,7 +6,7 @@ import type { ResolvedDesktopCaseOptions } from '@fluentui-react-native/desktop- import storyTests from '../../config/transform-story-tests.cjs'; -const { collectWdioTests, compileWdioTest } = storyTests; +const { collectWdioTests, compileWdioTest, rejectDirectInput } = storyTests; export type ExtractedWdioStory = { digest: string; @@ -26,11 +26,18 @@ export function extractWdioTests(code: string, sourceFile: string): ReadonlyMap< visitor: { Program(program: NodePath) { for (const [exportName, { callbacks, value }] of collectWdioTests(program)) { - const compiled = callbacks.map(({ callback, name, options }) => ({ - ...(name === undefined ? {} : { name }), - code: compileWdioTest(callback, sourceFile, code), - ...(options !== undefined ? { options: validateDesktopCaseOptions(options, `${sourceFile}.wdio`) } : {}), - })); + const compiled = callbacks.map(({ callback, name, options }) => { + const compiledCode = compileWdioTest(callback, sourceFile, code); + const descriptor = options === undefined ? undefined : validateDesktopCaseOptions(options, `${sourceFile}.wdio`); + if (descriptor !== undefined) { + rejectDirectInput(callback, { name, input: descriptor.input, inputSource: descriptor.inputSource }); + } + return { + ...(name === undefined ? {} : { name }), + code: compiledCode, + ...(descriptor === undefined ? {} : { options: descriptor }), + }; + }); const source = code.slice(value.node.start!, value.node.end!); tests.set(exportName, { tests: compiled, diff --git a/packages/agentic/storybook-desktop/src/testing/laneVerdict.test.ts b/packages/agentic/storybook-desktop/src/testing/laneVerdict.test.ts new file mode 100644 index 0000000000..845858dd7f --- /dev/null +++ b/packages/agentic/storybook-desktop/src/testing/laneVerdict.test.ts @@ -0,0 +1,75 @@ +import { assessWdioCohort, hasExecutedWdioCase, describeWdioCounts, type WdioCaseOutcome } from './laneVerdict.js'; + +const result = (testName: string, status: WdioCaseOutcome['status'], gap?: WdioCaseOutcome['gap']): WdioCaseOutcome => ({ + storyId: 'story', + testName, + status, + ...(gap ? { gap } : {}), +}); + +describe('WDIO cohort verdict', () => { + test('a compatibility cohort needs one executed test unless it is optional', () => { + expect(assessWdioCohort({ results: [result('a', 'passed'), result('b', 'skipped')], selected: 2 })).toMatchObject({ + executed: true, + incomplete: false, + }); + expect(assessWdioCohort({ results: [result('a', 'skipped')], selected: 1 })).toMatchObject({ + incomplete: true, + message: expect.stringContaining('no selected test executed'), + }); + expect(assessWdioCohort({ results: [result('a', 'unsupported')], selected: 1 })).toMatchObject({ incomplete: true }); + expect(assessWdioCohort({ results: [result('a', 'unsupported')], selected: 1, optional: true })).toMatchObject({ incomplete: false }); + expect(assessWdioCohort({ results: [result('a', 'passed'), result('b', 'unsupported')], selected: 2 })).toMatchObject({ + incomplete: false, + }); + }); + + test('a lane rejects a skip and an undeclared unsupported result, and names them', () => { + for (const status of ['skipped', 'unsupported'] as const) { + expect(assessWdioCohort({ results: [result('a', 'passed'), result('b', status)], selected: 2, lane: 'accessible' })).toMatchObject({ + executed: true, + incomplete: true, + message: expect.stringContaining('the accessible lane could not run 1 selected tests that have no declared gap (story / b)'), + }); + } + }); + + test('a lane accepts declared gaps and optional never masks an undeclared one', () => { + const declared = [result('a', 'passed'), result('b', 'unsupported', 'unsupported'), result('c', 'unsupported', 'quarantine')]; + expect(assessWdioCohort({ results: declared, selected: 3, lane: 'physical' })).toMatchObject({ incomplete: false }); + expect( + assessWdioCohort({ results: [result('a', 'unsupported', 'unsupported')], selected: 1, lane: 'physical', optional: true }), + ).toMatchObject({ executed: false, incomplete: false }); + expect(assessWdioCohort({ results: [result('a', 'unsupported')], selected: 1, lane: 'physical', optional: true })).toMatchObject({ + incomplete: true, + }); + expect(assessWdioCohort({ results: [result('a', 'unsupported', 'unsupported')], selected: 1, lane: 'physical' })).toMatchObject({ + incomplete: true, + message: expect.stringContaining('no selected test executed'), + }); + }); + + test('selected work that never finished is incomplete in every mode', () => { + expect(assessWdioCohort({ results: [result('a', 'passed')], selected: 3 })).toMatchObject({ + incomplete: true, + message: expect.stringContaining('2 of 3 selected tests did not finish'), + }); + }); + + test('counts only passes and failures as executed', () => { + expect(hasExecutedWdioCase([result('a', 'skipped'), result('b', 'unsupported')])).toBe(false); + expect(hasExecutedWdioCase([result('a', 'failed')])).toBe(true); + }); + + test('summarizes counts and breaks unsupported down only when present', () => { + expect(describeWdioCounts([result('a', 'passed'), result('b', 'skipped')])).toBe('1 passed, 1 skipped'); + expect( + describeWdioCounts([ + result('a', 'passed'), + result('b', 'unsupported', 'unsupported'), + result('c', 'unsupported', 'quarantine'), + result('d', 'unsupported'), + ]), + ).toBe('1 passed, 0 skipped, 3 unsupported (1 declared, 1 quarantined)'); + }); +}); diff --git a/packages/agentic/storybook-desktop/src/testing/laneVerdict.ts b/packages/agentic/storybook-desktop/src/testing/laneVerdict.ts new file mode 100644 index 0000000000..e33f24a8fc --- /dev/null +++ b/packages/agentic/storybook-desktop/src/testing/laneVerdict.ts @@ -0,0 +1,72 @@ +import type { DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; +import { describeDesktopUnsupported } from '@fluentui-react-native/desktop-test/reporting'; + +export type WdioCaseOutcome = { + storyId: string; + testName?: string; + status: 'passed' | 'failed' | 'skipped' | 'unsupported'; + gap?: 'unsupported' | 'quarantine'; +}; + +export type WdioCohortVerdict = { + /** At least one selected case ran to a pass or a failure. */ + executed: boolean; + incomplete: boolean; + /** Why the cohort is incomplete. */ + message?: string; +}; + +export function hasExecutedWdioCase(results: readonly Pick[]): boolean { + return results.some(({ status }) => status === 'passed' || status === 'failed'); +} + +/** Counts for a summary line. The unsupported breakdown appears only when something is unsupported, so other summaries are unchanged. */ +export function describeWdioCounts(results: readonly { status: string; gap?: WdioCaseOutcome['gap'] }[]): string { + const count = (status: string) => results.filter((result) => result.status === status).length; + const parts = [`${count('passed')} passed`, `${count('skipped')} skipped`]; + const unsupported = results.filter(({ status }) => status === 'unsupported'); + if (unsupported.length) { + parts.push( + describeDesktopUnsupported({ + unsupported: unsupported.length, + declaredUnsupported: unsupported.filter(({ gap }) => gap === 'unsupported').length, + quarantined: unsupported.filter(({ gap }) => gap === 'quarantine').length, + }), + ); + } + return parts.join(', '); +} + +/** + * A cohort is incomplete when selected work did not finish, or when nothing executed and it is not `optional`. In an + * input lane a run-time skip, or an `unsupported` result with no declared gap, is also incomplete, and `optional` never + * masks it: only a declaration may excuse a case the lane selected. A cohort that is all declared gaps is the caller's + * to judge, because a smoke run executes one story at a time and so cannot require every story to run something. + */ +export function assessWdioCohort(options: { + results: readonly WdioCaseOutcome[]; + selected: number; + lane?: DesktopInputLane; + optional?: boolean; +}): WdioCohortVerdict { + const { results, selected, lane, optional } = options; + const executed = hasExecutedWdioCase(results); + const undeclared = lane + ? results.filter(({ status, gap }) => status === 'skipped' || (status === 'unsupported' && gap === undefined)) + : []; + const reasons: string[] = []; + if (results.length !== selected) reasons.push(`${selected - results.length} of ${selected} selected tests did not finish`); + if (undeclared.length) { + const names = undeclared.slice(0, 5).map(({ storyId, testName }) => (testName === undefined ? storyId : `${storyId} / ${testName}`)); + const more = undeclared.length > names.length ? `, and ${undeclared.length - names.length} more` : ''; + reasons.push( + `the ${lane} lane could not run ${undeclared.length} selected tests that have no declared gap (${names.join(', ')}${more})`, + ); + } + if (!executed && !optional) reasons.push('no selected test executed'); + return { + executed, + incomplete: reasons.length > 0, + ...(reasons.length ? { message: `Required executable story cohort is incomplete: ${reasons.join('; ')}.` } : {}), + }; +} diff --git a/packages/agentic/storybook-desktop/src/testing/runWdioTests.test.ts b/packages/agentic/storybook-desktop/src/testing/runWdioTests.test.ts deleted file mode 100644 index 5860dd8e4f..0000000000 --- a/packages/agentic/storybook-desktop/src/testing/runWdioTests.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { validateDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; -import type { DesktopStoryManifest } from '@fluentui-react-native/desktop-driver'; - -import { missingRequirements, resolveCaseClickMode, selectWdioCases, selectWdioStories } from './runWdioTests.js'; - -describe('WDIO input lane selection', () => { - const manifest: DesktopStoryManifest = { - endpoint: 'win32', - entries: [ - { - id: 'button', - name: 'Button', - packageName: 'components', - sourcePath: 'button.stories.tsx', - tags: ['story'], - title: 'Button', - wdio: { - digest: 'digest', - exportName: 'Default', - testNames: ['adaptive', 'accessible', 'win32-only'], - cases: [ - { name: 'adaptive', options: validateDesktopCaseOptions({ requires: ['click'] }) }, - { name: 'accessible', options: validateDesktopCaseOptions({ requires: ['accessibility-click'] }) }, - { name: 'win32-only', options: validateDesktopCaseOptions({ platforms: ['win32'] }) }, - ], - }, - }, - ], - platformManifestDigest: 'digest', - portablePlanDigest: 'digest', - schemaVersion: 1, - }; - - test('selects by modality and target platform', () => { - expect(selectWdioStories(manifest, { input: 'accessible' }, 'win32').map(({ id }) => id)).toEqual(['button']); - expect(selectWdioCases(manifest.entries[0], { input: 'accessible' }, 'win32')?.map(({ name }) => name)).toEqual([ - 'adaptive', - 'accessible', - 'win32-only', - ]); - expect(selectWdioCases(manifest.entries[0], {}, 'macos')?.map(({ name }) => name)).toEqual(['adaptive', 'accessible']); - expect(selectWdioCases(manifest.entries[0], { input: 'physical' }, 'win32')?.map(({ name }) => name)).toEqual([ - 'adaptive', - 'win32-only', - ]); - expect(selectWdioCases(manifest.entries[0], { input: 'physical', test: 'accessible' }, 'macos')).toEqual([]); - expect(selectWdioCases(manifest.entries[0], { input: 'physical', test: 'win32-only' }, 'macos')).toEqual([]); - }); - - test('rejects legacy callback entries in a lane', () => { - const legacy = { - ...manifest, - entries: [{ ...manifest.entries[0], wdio: { digest: 'legacy', exportName: 'Default', testNames: ['legacy'] } }], - } satisfies DesktopStoryManifest; - expect(() => selectWdioStories(legacy, { input: 'physical' }, 'win32')).toThrow('declare it with desktopIt'); - }); - - test('adapts click requirements to each lane without fallback', () => { - const descriptor = validateDesktopCaseOptions({ requires: ['click'] }); - const accessibleMode = resolveCaseClickMode(descriptor, 'auto', 'accessible'); - const physicalMode = resolveCaseClickMode(descriptor, 'auto', 'physical'); - expect(accessibleMode).toBe('accessibility'); - expect(physicalMode).toBe('physical'); - expect(missingRequirements(descriptor, { accessibilityClick: true, physicalClick: false }, physicalMode)).toEqual(['click']); - expect(missingRequirements(descriptor, { accessibilityClick: false, physicalClick: true }, accessibleMode)).toEqual(['click']); - expect(missingRequirements(descriptor, { accessibilityClick: true }, accessibleMode)).toEqual([]); - }); -}); diff --git a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts index 8ed1c3a371..524bd88ae3 100644 --- a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts +++ b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts @@ -3,9 +3,13 @@ import { createHash } from 'node:crypto'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; -import type { DesktopStoryManifest, DesktopStoryManifestEntry } from '@fluentui-react-native/desktop-driver'; -import { matchesDesktopPlatforms } from '@fluentui-react-native/desktop-driver/authoring'; -import type { DesktopArtifact, DesktopInputLane, ResolvedDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopStoryManifest } from '@fluentui-react-native/desktop-driver'; +import { + missingDesktopRequirements, + resolveDesktopClickMode, + unmaskedAccessibleFeatures, + type DesktopArtifact, +} from '@fluentui-react-native/desktop-driver/authoring'; import { ArtifactManager, captureDesktopFailureEvidence } from '@fluentui-react-native/desktop-driver/artifacts'; import { connectDesktopWebdriver } from '@fluentui-react-native/desktop-driver/wdio'; import type { DesktopReportWriter } from '@fluentui-react-native/desktop-test/reporting'; @@ -16,6 +20,15 @@ import type { Platforms } from '../config/platforms.js'; import { resolveWdioOptions, type DesktopStorybookWdioOptions } from '../config/wdio.js'; import { NodeDesktopCommandRunner, type DesktopCommandRunner, type PreparedDesktopCommand } from '../cli/commandRunner.js'; import { extractWdioTests } from './extractWdioTests.js'; +import { assessWdioCohort } from './laneVerdict.js'; +import { + csfTestId, + decideWdioSelection, + describeWdioExclusions, + matchesWdioTest, + plannedWdioWork, + type WdioSelectionContext, +} from './selection.js'; import type { WdioWorkerOptions } from './worker.js'; import { formatDesktopStorybookError, writeDesktopStorybookFailure, type DesktopStorybookErrorOutput } from '../../config/diagnostics.cjs'; @@ -35,6 +48,8 @@ export type RunWdioStoryTestsOptions = DesktopStorybookWdioOptions & { config: DesktopStorybookConfig; manifest: DesktopStoryManifest; platform: Platforms; + /** The scope domains and values of this target, for case `scope` selection. */ + scope?: WdioSelectionContext['scope']; targetId: string; url: string; artifactsRoot?: string; @@ -42,6 +57,8 @@ export type RunWdioStoryTestsOptions = DesktopStorybookWdioOptions & { report?: DesktopReportWriter; signal?: AbortSignal; optional?: boolean; + /** The caller already reported the selection and announced the planned work, so this call must not repeat it. */ + announced?: boolean; }; export class WdioStoryTestRunError extends Error { @@ -61,36 +78,13 @@ class MissingRequirementsError extends Error { } } -export function selectWdioStories( - manifest: DesktopStoryManifest, - options: DesktopStorybookWdioOptions, - platform?: Platforms, -): readonly DesktopStoryManifestEntry[] { - return manifest.entries - .filter((entry) => entry.wdio && (!options.story || path.matchesGlob(entry.id, options.story))) - .filter((entry) => !options.tag || entry.tags.includes(options.tag)) - .filter((entry) => { - if (options.input || entry.wdio?.cases) return (selectWdioCases(entry, options, platform)?.length ?? 0) > 0; - return (entry.wdio?.testNames ?? ['default']).some((name) => matchesWdioTest(name, options.test)); - }) - .sort((left, right) => left.id.localeCompare(right.id)); -} - -export function selectWdioCases( - entry: DesktopStoryManifestEntry, - options: DesktopStorybookWdioOptions, - platform?: Platforms, -): NonNullable['cases'] { - if (!entry.wdio?.cases?.length) { - if (!options.input) return undefined; - throw new Error(`Input lane "${options.input}" cannot run legacy wdio callback in "${entry.id}"; declare it with desktopIt.`); +/** The driver cannot show it disabled physical input, so an accessible lane could not keep its guarantee. */ +class UnmaskedLaneError extends Error { + constructor(unmasked: readonly string[]) { + super( + `The accessible input lane needs a driver that reports ${unmasked.join(', ')} disabled, and this one does not. Start it with "driver --input accessible".`, + ); } - return entry.wdio.cases.filter( - ({ name, options: descriptor }) => - matchesWdioTest(name, options.test) && - (!platform || matchesDesktopPlatforms(descriptor.platforms, platform)) && - (!options.input || descriptor.input === 'both' || descriptor.input === options.input), - ); } export async function runWdioStoryTests( @@ -98,31 +92,37 @@ export async function runWdioStoryTests( runner?: DesktopCommandRunner, ): Promise { const settings = resolveWdioOptions(options); - const stories = selectWdioStories(options.manifest, settings, options.platform); - if (stories.length === 0) { - throw new Error('No executable wdio story tests matched the requested selection.'); + const selection = decideWdioSelection(options.manifest, settings, { platform: options.platform, scope: options.scope }); + const planned = plannedWdioWork(selection, settings); + const { platform: byPlatform, scope: byScope, input: byInput } = selection.excluded; + // Reported before any empty-selection error so the exclusions are accounted for. A lane always states them, + // even when nothing was excluded; compatibility mode only when it excluded something. + if (!options.announced && options.report && (settings.input || byPlatform + byScope + byInput > 0)) { + options.report.emit({ kind: 'selection', payload: { excluded: selection.excluded } }); + } + if (selection.stories.length === 0) { + throw new Error(`No executable wdio story tests matched the requested selection${describeWdioExclusions(selection)}.`); } - const cases = stories.flatMap((entry) => { + const cases = selection.stories.flatMap((entry) => { const sourceFile = path.join(options.config.resolvePackage(entry.packageName).root, entry.sourcePath); const extracted = extractWdioTests(fs.readFileSync(sourceFile, 'utf8'), sourceFile).get(entry.wdio!.exportName); if (!extracted || extracted.digest !== entry.wdio!.digest) { throw new Error(`Executable test source changed for ${entry.id}; regenerate the Storybook manifest.`); } - const selectedCases = selectWdioCases(entry, settings, options.platform); + const declared = selection.selected.get(entry.id); return extracted.tests .filter(({ name }) => matchesWdioTest(name ?? 'default', settings.test)) - .map(({ name, code }) => ({ - entry, - code, - testName: name, - descriptor: entry.wdio?.cases?.find((testCase) => testCase.name === name)?.options, - })) - .filter(({ testName }) => { - if (!selectedCases) return !settings.input; - return selectedCases.some(({ name }) => name === testName); + .flatMap(({ name, code }) => { + const decision = name === undefined ? undefined : declared?.get(name); + if (declared && !decision) return []; + return [{ entry, code, testName: name, descriptor: decision?.descriptor, gap: decision?.gap }]; }); }); if (cases.length === 0) throw new Error('No executable wdio tests matched the requested selection.'); + const plannedIds = new Set(planned.map(({ workId }) => workId)); + if (cases.length !== plannedIds.size || cases.some(({ entry, testName }) => !plannedIds.has(csfTestId(entry.id, testName)))) { + throw new Error('The executable tests differ from the selected manifest cases; regenerate the Storybook manifest.'); + } const artifacts = new ArtifactManager( options.artifactsRoot ?? (options.report @@ -135,11 +135,7 @@ export async function runWdioStoryTests( ).href; const results: WdioStoryTestResult[] = []; const executionFailures: unknown[] = []; - for (const { entry, testName } of cases) - options.report?.emit({ - kind: 'work-selected', - payload: { workId: csfTestId(entry.id, testName), label: `${entry.id} / ${testName ?? 'default'}` }, - }); + if (!options.announced) for (const payload of planned) options.report?.emit({ kind: 'work-selected', payload }); const ownedRunner = runner ? undefined : new NodeDesktopCommandRunner({ reporter: options.report }); const commands = runner ?? ownedRunner!; let directory: string | undefined; @@ -151,7 +147,7 @@ export async function runWdioStoryTests( try { fs.mkdirSync(generatedRoot, { recursive: true }); directory = fs.mkdtempSync(path.join(generatedRoot, 'run-')); - for (const [index, { entry, code, testName, descriptor }] of cases.entries()) { + for (const [index, { entry, code, testName, descriptor, gap }] of cases.entries()) { options.signal?.throwIfAborted(); const displayName = testName === undefined ? entry.id : `${entry.id} / ${testName}`; const artifactId = @@ -171,7 +167,6 @@ export async function runWdioStoryTests( const context = { story: entry.id, test: testName ?? 'default', endpoint: options.platform }; options.report?.emit({ kind: 'scope-start', payload: { label: displayName, category: 'worker' } }, testId, context, testId); options.report?.emit({ kind: 'test-start', payload: { testId, title: displayName, attempt: 0 } }, testId, context); - const gap = descriptor && declaredGapFor(descriptor, options.platform, settings.input); if (gap) { result.status = 'unsupported'; result.skipReason = gap.reason; @@ -181,7 +176,15 @@ export async function runWdioStoryTests( options.report?.emit( { kind: 'test-result', - payload: { testId, title: displayName, attempt: 0, status: 'unsupported', durationMs: result.durationMs, reason: gap.reason }, + payload: { + testId, + title: displayName, + attempt: 0, + status: 'unsupported', + durationMs: result.durationMs, + reason: gap.reason, + gap: gap.kind, + }, }, testId, context, @@ -190,7 +193,7 @@ export async function runWdioStoryTests( activeResult = undefined; continue; } - const clickMode = resolveCaseClickMode(descriptor, settings.clickMode, settings.input); + const clickMode = resolveDesktopClickMode(descriptor, settings.clickMode, settings.input); // The supervisor owns the session, so even a crashed or timed-out Node worker cannot strand it. const desktop = await connectDesktopWebdriver({ clickMode, @@ -206,7 +209,13 @@ export async function runWdioStoryTests( if (live.endpoint !== options.platform || live.platformManifestDigest !== options.manifest.platformManifestDigest) { throw new Error('The running Storybook manifest is stale or targets a different platform. Restart storybook driver and the app.'); } - const missing = missingRequirements(descriptor, desktop.browser.capabilities['furn:features'], clickMode); + if (settings.input === 'accessible') { + // A driver that cannot report its physical input disabled fails closed rather than silently running unmasked. + phase = 'lane verification'; + const unmasked = unmaskedAccessibleFeatures(desktop.browser.capabilities['furn:features']); + if (unmasked.length) throw new UnmaskedLaneError(unmasked); + } + const missing = missingDesktopRequirements(descriptor, desktop.browser.capabilities['furn:features'], clickMode); if (missing.length) throw new MissingRequirementsError(missing); phase = 'navigation'; await desktop.openStory(entry.id); @@ -263,6 +272,8 @@ export async function runWdioStoryTests( if (error instanceof MissingRequirementsError) { result.status = 'unsupported'; result.skipReason = error.message; + } else if (error instanceof UnmaskedLaneError) { + failure = error; } else { failure = error; executionFailures.push(error); @@ -298,6 +309,11 @@ export async function runWdioStoryTests( `Could not release the wdio session for ${displayName}; stopping before another test starts.`, ); } + // Every case would fail the same way, so stop here and let the run report it once, against this case. + if (failure instanceof UnmaskedLaneError) { + phase = 'lane verification'; + throw failure; + } result.durationMs = Date.now() - started; results.push(result); options.report?.emit( @@ -348,14 +364,9 @@ export async function runWdioStoryTests( } const failures: unknown[] = infrastructureFailure === undefined ? [] : [infrastructureFailure]; failures.push(...executionFailures); - const allSkipped = results.every(({ status }) => status === 'skipped'); - const laneIncomplete = - !!settings.input && - (!results.some(({ status }) => status === 'passed' || status === 'failed') || - results.some(({ status, gap }) => status === 'skipped' || (status === 'unsupported' && gap === undefined))); - const incomplete = results.length !== cases.length || (allSkipped && !options.optional) || laneIncomplete; - if (incomplete && failures.length === 0) - failures.push(new Error('Required executable story cohort did not execute any tests or left selected work unfinished.')); + const verdict = assessWdioCohort({ results, selected: cases.length, lane: settings.input, optional: options.optional }); + const incomplete = verdict.incomplete; + if (incomplete && failures.length === 0) failures.push(new Error(verdict.message)); try { const reportPath = artifacts.writeMetadata('run', { endpoint: options.platform, @@ -364,9 +375,9 @@ export async function runWdioStoryTests( ? incomplete && executionFailures.length === 0 && infrastructureFailure === undefined ? 'incomplete' : 'failed' - : allSkipped - ? 'skipped' - : 'passed', + : verdict.executed + ? 'passed' + : 'skipped', ...(infrastructureFailure === undefined ? {} : { error: errorMessage(infrastructureFailure) }), tests: results, }); @@ -407,85 +418,6 @@ function isWorkerResult(value: unknown): value is WorkerResult { ); } -export function matchesWdioTest(name: string, pattern?: string): boolean { - return pattern === undefined || path.matchesGlob(name, pattern); -} - -function csfTestId(storyId: string, testName?: string): string { - return `csf-${createHash('sha256') - .update(JSON.stringify([storyId, testName ?? 'default'])) - .digest('hex')}`; -} - -export function resolveCaseClickMode( - descriptor: ResolvedDesktopCaseOptions | undefined, - configured: 'auto' | 'physical' | 'accessibility', - input: DesktopInputLane | undefined, -): 'auto' | 'physical' | 'accessibility' { - if (input === 'accessible') return 'accessibility'; - if (input === 'physical') return 'physical'; - if (descriptor?.input === 'accessible' || descriptor?.requires?.includes('accessibility-click')) return 'accessibility'; - if (descriptor?.input === 'physical') return 'physical'; - return configured; -} - -export function missingRequirements( - descriptor: ResolvedDesktopCaseOptions | undefined, - features: Readonly> | undefined, - clickMode: 'auto' | 'physical' | 'accessibility', -): string[] { - if (!descriptor?.requires?.length) return []; - const missing: string[] = []; - for (const requirement of descriptor.requires) { - const feature = - requirement === 'click' - ? clickMode === 'physical' - ? 'physicalClick' - : clickMode === 'accessibility' - ? 'accessibilityClick' - : undefined - : requirement === 'physical-click' - ? 'physicalClick' - : requirement === 'accessibility-click' - ? 'accessibilityClick' - : requirement === 'accessibility-actions' - ? 'accessibilityActions' - : requirement === 'element-screenshot' - ? 'elementScreenshot' - : requirement === 'set-window-rect' - ? 'setWindowRect' - : requirement; - if (requirement === 'click' && clickMode === 'auto') { - if (!features?.physicalClick && !features?.accessibilityClick) missing.push(requirement); - } else if (feature && features?.[feature] !== true) { - missing.push(requirement); - } - } - return missing; -} - -function declaredGapFor( - descriptor: ResolvedDesktopCaseOptions, - endpoint: Platforms, - input: DesktopInputLane | undefined, -): { kind: 'unsupported' | 'quarantine'; reason: string } | undefined { - if (descriptor.unsupported && matchesDesktopPlatforms(descriptor.unsupported.platforms, endpoint)) { - return { kind: 'unsupported', reason: descriptor.unsupported.reason }; - } - const quarantine = descriptor.quarantine; - if ( - quarantine && - (!quarantine.platforms || matchesDesktopPlatforms(quarantine.platforms, endpoint)) && - (!quarantine.input || quarantine.input === input) - ) { - if (Date.parse(`${quarantine.expires}T00:00:00Z`) < Date.now()) { - throw new Error(`Quarantine ${quarantine.issue} expired on ${quarantine.expires}.`); - } - return { kind: 'quarantine', reason: `${quarantine.issue}: ${quarantine.reason}` }; - } - return undefined; -} - function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } diff --git a/packages/agentic/storybook-desktop/src/testing/selection.test.ts b/packages/agentic/storybook-desktop/src/testing/selection.test.ts new file mode 100644 index 0000000000..847a27062e --- /dev/null +++ b/packages/agentic/storybook-desktop/src/testing/selection.test.ts @@ -0,0 +1,301 @@ +import type { DesktopStoryManifest, DesktopStoryManifestEntry } from '@fluentui-react-native/desktop-driver'; +import { resolveDesktopScopeDomains, validateDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; + +import { + csfTestId, + decideWdioSelection, + describeWdioExclusions, + listWdioSelection, + plannedWdioWork, + type WdioSelection, +} from './selection.js'; + +type DeclaredCase = readonly [name: string, options: unknown]; + +function declared(id: string, cases: readonly DeclaredCase[], tags: string[] = ['story']): DesktopStoryManifestEntry { + return { + id, + name: id, + packageName: 'components', + sourcePath: `${id}.stories.tsx`, + tags, + title: id, + wdio: { + digest: 'digest', + exportName: 'Default', + testNames: cases.map(([name]) => name), + cases: cases.map(([name, options]) => ({ name, options: validateDesktopCaseOptions(options) })), + }, + }; +} + +function legacy(id: string, testNames?: string[]): DesktopStoryManifestEntry { + return { + id, + name: id, + packageName: 'components', + sourcePath: `${id}.stories.tsx`, + tags: ['story'], + title: id, + wdio: { digest: 'legacy', exportName: 'Default', ...(testNames ? { testNames } : {}) }, + }; +} + +function manifestOf(...entries: DesktopStoryManifestEntry[]): DesktopStoryManifest { + return { endpoint: 'win32', entries, platformManifestDigest: 'digest', portablePlanDigest: 'digest', schemaVersion: 1 }; +} + +const selectedNames = (selection: WdioSelection, id = 'button') => [...(selection.selected.get(id)?.keys() ?? [])]; + +const button = declared('button', [ + ['adaptive', { requires: ['click'] }], + ['accessible', { requires: ['accessibility-click'] }], + ['win32-only', { platforms: ['win32'] }], +]); + +describe('WDIO selection', () => { + test('selects by input lane and target platform', () => { + const manifest = manifestOf(button); + expect(selectedNames(decideWdioSelection(manifest, { input: 'accessible' }, { platform: 'win32' }))).toEqual([ + 'adaptive', + 'accessible', + 'win32-only', + ]); + expect(selectedNames(decideWdioSelection(manifest, {}, { platform: 'macos' }))).toEqual(['adaptive', 'accessible']); + expect(selectedNames(decideWdioSelection(manifest, { input: 'physical' }, { platform: 'win32' }))).toEqual(['adaptive', 'win32-only']); + }); + + test('records every decision with its first exclusion reason', () => { + const selection = decideWdioSelection(manifestOf(button), { input: 'physical' }, { platform: 'macos' }); + expect(selection.decisions.map(({ name, excluded }) => [name, excluded])).toEqual([ + ['adaptive', undefined], + ['accessible', 'input'], + ['win32-only', 'platform'], + ]); + expect(selection.excluded).toEqual({ platform: 1, scope: 0, input: 1 }); + }); + + test('counts exclusions only among the cases the user filters selected', () => { + const manifest = manifestOf(button, declared('other', [['unrelated', { platforms: ['win32'] }]])); + const byTest = decideWdioSelection(manifest, { input: 'physical', test: 'accessible' }, { platform: 'macos' }); + expect(byTest.stories).toEqual([]); + expect(byTest.excluded).toEqual({ platform: 0, scope: 0, input: 1 }); + const byStory = decideWdioSelection(manifest, { input: 'physical', story: 'button', test: 'win32-only' }, { platform: 'macos' }); + expect(byStory.excluded).toEqual({ platform: 1, scope: 0, input: 0 }); + expect(describeWdioExclusions(byStory)).toBe(' (excluded 1 by platform, 0 by scope, 0 by input)'); + expect(describeWdioExclusions(decideWdioSelection(manifest, {}, { platform: 'win32' }))).toBe(''); + }); + + test('excludes by platform, then scope, then input', () => { + const layered = manifestOf( + declared('layered', [['all', { platforms: ['win32'], scope: { 'rn.renderer': ['paper'] }, requires: ['keyboard'] }]]), + ); + const decide = (platform: 'macos' | 'win32', renderer: string) => + decideWdioSelection( + layered, + { input: 'accessible' }, + { platform, scope: { domains: resolveDesktopScopeDomains(), values: { 'rn.renderer': renderer } } }, + ); + expect(decide('macos', 'fabric').excluded).toEqual({ platform: 1, scope: 0, input: 0 }); + expect(decide('win32', 'fabric').excluded).toEqual({ platform: 0, scope: 1, input: 0 }); + expect(decide('win32', 'paper').excluded).toEqual({ platform: 0, scope: 0, input: 1 }); + }); + + test('scopes cases by a value the target holds, including custom domains', () => { + const domains = resolveDesktopScopeDomains({ 'product.flavor': ['alpha', 'beta'] }); + const scoped = manifestOf( + declared('scoped', [ + ['paper-only', { scope: { 'rn.renderer': ['paper'] } }], + ['beta-only', { scope: { 'product.flavor': ['beta'] } }], + ['both-keys', { scope: { 'rn.renderer': ['paper', 'fabric'], 'product.flavor': ['alpha'] } }], + ]), + ); + const values = (renderer: string, flavor: string) => ({ domains, values: { 'rn.renderer': renderer, 'product.flavor': flavor } }); + expect(selectedNames(decideWdioSelection(scoped, {}, { platform: 'win32', scope: values('paper', 'beta') }), 'scoped')).toEqual([ + 'paper-only', + 'beta-only', + ]); + expect(selectedNames(decideWdioSelection(scoped, {}, { platform: 'win32', scope: values('fabric', 'alpha') }), 'scoped')).toEqual([ + 'both-keys', + ]); + }); + + test('treats a scope key the target does not supply as a configuration error rather than an exclusion', () => { + const scoped = manifestOf(declared('scoped', [['paper-only', { scope: { 'rn.renderer': ['paper'] } }]])); + expect(() => decideWdioSelection(scoped, {}, { platform: 'win32' })).toThrow('is not supplied by this target'); + const custom = manifestOf(declared('scoped', [['flavored', { scope: { 'product.flavor': ['beta'] } }]])); + expect(() => + decideWdioSelection(custom, {}, { platform: 'win32', scope: { domains: resolveDesktopScopeDomains(), values: {} } }), + ).toThrow('unknown scope key "product.flavor"'); + const outside = manifestOf(declared('scoped', [['bogus', { scope: { 'rn.renderer': ['bogus'] } }]])); + expect(() => + decideWdioSelection( + outside, + {}, + { platform: 'win32', scope: { domains: resolveDesktopScopeDomains(), values: { 'rn.renderer': 'paper' } } }, + ), + ).toThrow('is not one of: fabric, paper'); + }); + + test('does not evaluate the scope of a case another platform already excluded', () => { + const scoped = manifestOf(declared('scoped', [['win32-paper', { platforms: ['win32'], scope: { 'rn.renderer': ['paper'] } }]])); + expect(decideWdioSelection(scoped, {}, { platform: 'macos' }).excluded).toEqual({ platform: 1, scope: 0, input: 0 }); + }); + + test('filters by story glob and tag and orders stories by id', () => { + const manifest = manifestOf( + declared('zeta', [['a', {}]], ['wide']), + declared('alpha', [['a', {}]], ['wide']), + declared('middle', [['a', {}]], ['narrow']), + ); + expect(decideWdioSelection(manifest, {}, { platform: 'win32' }).stories.map(({ id }) => id)).toEqual(['alpha', 'middle', 'zeta']); + expect(decideWdioSelection(manifest, { tag: 'wide' }, { platform: 'win32' }).stories.map(({ id }) => id)).toEqual(['alpha', 'zeta']); + expect(decideWdioSelection(manifest, { story: 'm*' }, { platform: 'win32' }).stories.map(({ id }) => id)).toEqual(['middle']); + }); + + describe('legacy callbacks', () => { + test('run in compatibility mode when a name matches the test filter', () => { + const manifest = manifestOf(legacy('plain'), legacy('named', ['first', 'second']), declared('button', [['a', {}]])); + expect(decideWdioSelection(manifest, {}, { platform: 'win32' }).stories.map(({ id }) => id)).toEqual(['button', 'named', 'plain']); + expect(decideWdioSelection(manifest, { test: 'sec*' }, { platform: 'win32' }).stories.map(({ id }) => id)).toEqual(['named']); + expect(decideWdioSelection(manifest, { test: 'nothing' }, { platform: 'win32' }).stories).toEqual([]); + }); + + test('are rejected together in a lane so one pass migrates them', () => { + const manifest = manifestOf(legacy('b-legacy'), legacy('a-legacy', ['first']), declared('button', [['a', {}]])); + expect(() => decideWdioSelection(manifest, { input: 'physical' }, { platform: 'win32' })).toThrow( + 'Input lane "physical" cannot run legacy wdio callbacks; declare them with desktopIt: a-legacy, b-legacy.', + ); + }); + + test('are ignored in a lane when the test filter names nothing they define, and summarized past ten', () => { + const many = Array.from({ length: 12 }, (_, index) => legacy(`legacy-${String(index).padStart(2, '0')}`)); + expect(() => decideWdioSelection(manifestOf(...many), { input: 'accessible' }, { platform: 'win32' })).toThrow( + /legacy-00, legacy-01.*legacy-09, and 2 more\.$/, + ); + expect(decideWdioSelection(manifestOf(...many), { input: 'accessible', test: 'absent' }, { platform: 'win32' }).stories).toEqual([]); + }); + }); + + describe('declared gaps', () => { + const gapped = manifestOf( + declared('gapped', [ + ['unsupported', { unsupported: { platforms: ['win32'], reason: 'No host support.' } }], + ['quarantined', { quarantine: { issue: '#9', reason: 'flaky', expires: '2026-01-01' } }], + ['physical-quarantine', { quarantine: { issue: '#10', reason: 'drifts', expires: '2026-01-01', input: 'physical' } }], + ]), + ); + const before = Date.parse('2025-12-31T00:00:00Z'); + + test('are resolved at selection time for the cases that run', () => { + const win32 = decideWdioSelection(gapped, { input: 'physical' }, { platform: 'win32', now: before }); + expect([...win32.selected.get('gapped')!.values()].map(({ name, gap }) => [name, gap])).toEqual([ + ['unsupported', { kind: 'unsupported', reason: 'No host support.' }], + ['quarantined', { kind: 'quarantine', reason: '#9: flaky' }], + ['physical-quarantine', { kind: 'quarantine', reason: '#10: drifts' }], + ]); + const macos = decideWdioSelection(gapped, { input: 'accessible' }, { platform: 'macos', now: before }); + expect([...macos.selected.get('gapped')!.values()].map(({ name, gap }) => [name, gap?.kind])).toEqual([ + ['unsupported', undefined], + ['quarantined', 'quarantine'], + ['physical-quarantine', undefined], + ]); + }); + + test('stop the run before anything executes once a quarantine expires', () => { + expect(() => decideWdioSelection(gapped, {}, { platform: 'win32', now: Date.parse('2026-06-01T00:00:00Z') })).toThrow( + 'Quarantine #9 expired on 2026-01-01.', + ); + }); + + test('are not evaluated for a case the lane already excluded', () => { + const excludedByLane = manifestOf( + declared('gapped', [ + ['accessible', { requires: ['accessibility-click'], quarantine: { issue: '#9', reason: 'old', expires: '2020-01-01' } }], + ]), + ); + expect(decideWdioSelection(excludedByLane, { input: 'physical' }, { platform: 'win32' }).excluded.input).toBe(1); + }); + }); + + test('plans the work each selected case announces, legacy included', () => { + const manifest = manifestOf( + declared('button', [ + ['adaptive', {}], + ['physical', { requires: ['keyboard'] }], + ]), + legacy('named', ['x', 'y']), + ); + const planned = plannedWdioWork(decideWdioSelection(manifest, {}, { platform: 'win32' }), {}); + expect(planned).toEqual([ + { workId: csfTestId('button', 'adaptive'), label: 'button / adaptive' }, + { workId: csfTestId('button', 'physical'), label: 'button / physical' }, + { workId: csfTestId('named', 'x'), label: 'named / x' }, + { workId: csfTestId('named', 'y'), label: 'named / y' }, + ]); + const filtered = plannedWdioWork(decideWdioSelection(manifest, { test: 'y' }, { platform: 'win32' }), { test: 'y' }); + expect(filtered.map(({ label }) => label)).toEqual(['named / y']); + const lane = plannedWdioWork(decideWdioSelection(manifestOf(manifest.entries[0]), { input: 'accessible' }, { platform: 'win32' }), {}); + expect(lane.map(({ label }) => label)).toEqual(['button / adaptive']); + }); + + test('names a legacy callback without a declared name "default"', () => { + const planned = plannedWdioWork(decideWdioSelection(manifestOf(legacy('plain')), {}, { platform: 'win32' }), {}); + expect(planned).toEqual([{ workId: csfTestId('plain', 'default'), label: 'plain / default' }]); + expect(csfTestId('plain')).toBe(csfTestId('plain', 'default')); + }); + + describe('listing', () => { + const listed = manifestOf( + declared('button', [ + ['adaptive', { requires: ['click'] }], + ['physical', { requires: ['keyboard'], platforms: ['win32'] }], + ['accessible', { requires: ['accessibility-click'], unsupported: { platforms: ['win32'], reason: 'No host support.' } }], + ]), + legacy('named', ['first', 'second']), + legacy('plain'), + declared('only-physical', [['physical', { requires: ['keyboard'] }]]), + ); + + test('shows the selected cases with their gaps and the excluded cases with their first reason', () => { + const selection = decideWdioSelection(listed, { input: 'accessible', story: 'button' }, { platform: 'win32' }); + expect(selection.stories.map(({ id }) => id)).toEqual(['button']); + expect(listWdioSelection(selection, {})).toEqual([ + { + id: 'button', + sourcePath: 'button.stories.tsx', + tests: [ + { name: 'adaptive', input: 'both', inputSource: 'derived', requires: ['click'] }, + { + name: 'accessible', + input: 'accessible', + inputSource: 'derived', + requires: ['accessibility-click'], + gap: { kind: 'unsupported', reason: 'No host support.' }, + }, + ], + excluded: [ + { name: 'physical', input: 'physical', inputSource: 'derived', requires: ['keyboard'], platforms: ['win32'], reason: 'input' }, + ], + }, + ]); + }); + + test('lists a legacy callback by its names, or alone when it names none', () => { + const selection = decideWdioSelection(listed, { story: '{named,plain}', test: 'sec*' }, { platform: 'macos' }); + expect(listWdioSelection(selection, { test: 'sec*' })).toEqual([{ id: 'named', sourcePath: 'named.stories.tsx', tests: ['second'] }]); + expect(listWdioSelection(decideWdioSelection(listed, { story: 'plain' }, { platform: 'macos' }), {})).toEqual([ + { id: 'plain', sourcePath: 'plain.stories.tsx' }, + ]); + }); + + test('omits excluded when every matching case runs', () => { + const [story] = listWdioSelection(decideWdioSelection(listed, { story: 'only-physical' }, { platform: 'win32' }), {}); + expect(story).toEqual({ + id: 'only-physical', + sourcePath: 'only-physical.stories.tsx', + tests: [{ name: 'physical', input: 'physical', inputSource: 'derived', requires: ['keyboard'] }], + }); + }); + }); +}); diff --git a/packages/agentic/storybook-desktop/src/testing/selection.ts b/packages/agentic/storybook-desktop/src/testing/selection.ts new file mode 100644 index 0000000000..e4ae0f18a5 --- /dev/null +++ b/packages/agentic/storybook-desktop/src/testing/selection.ts @@ -0,0 +1,194 @@ +import { createHash } from 'node:crypto'; +import path from 'node:path'; + +import type { DesktopStoryManifest, DesktopStoryManifestEntry } from '@fluentui-react-native/desktop-driver'; +import { decideDesktopCase, resolveDesktopScopeDomains } from '@fluentui-react-native/desktop-driver/authoring'; +import type { + DesktopCaseTarget, + DesktopDeclaredGap, + DesktopScopeDomains, + DesktopScopeValues, + ResolvedDesktopCaseOptions, +} from '@fluentui-react-native/desktop-driver/authoring'; +import type { SelectionExclusions } from '@fluentui-react-native/desktop-test/reporting'; + +import type { Platforms } from '../config/platforms.js'; +import type { DesktopStorybookWdioOptions } from '../config/wdio.js'; + +export type WdioSelectionContext = { + platform: Platforms; + /** The domains and the values this target holds. Without it only the built-in domains exist and no key is supplied. */ + scope?: { domains: DesktopScopeDomains; values: DesktopScopeValues }; + /** Clock for quarantine expiry. */ + now?: number; +}; + +export type WdioDeclaredGap = DesktopDeclaredGap; + +/** One declared case that matched the user's filters. A case with no `excluded` reason is part of the run. */ +export type WdioCaseDecision = { + storyId: string; + name: string; + descriptor: ResolvedDesktopCaseOptions; + /** The first reason, in platform, scope, input order, that this target or lane does not run the case. */ + excluded?: keyof SelectionExclusions; + /** Declared `unsupported` or quarantine; the case is selected but not executed. */ + gap?: WdioDeclaredGap; +}; + +export type WdioSelection = { + /** Stories with at least one selected case, or a legacy callback that compatibility mode may run. */ + stories: readonly DesktopStoryManifestEntry[]; + /** Every declared case that matched the user's filters, in story and source order. */ + decisions: readonly WdioCaseDecision[]; + /** The selected declared cases by story and name. Legacy callbacks are absent. */ + selected: ReadonlyMap>; + /** Only cases that matched the user's filters count, so a narrow `--story` is not blamed for what it never asked for. */ + excluded: SelectionExclusions; +}; + +/** + * Decides every case before any session exists. Legacy callbacks run in compatibility mode and are an error in a lane, + * where each is listed so they can be migrated in one pass. Declared gaps are resolved here too, so an expired + * quarantine stops the run before anything executes. + */ +export function decideWdioSelection( + manifest: DesktopStoryManifest, + options: DesktopStorybookWdioOptions, + context: WdioSelectionContext, +): WdioSelection { + const domains = context.scope?.domains ?? resolveDesktopScopeDomains(); + const target: DesktopCaseTarget = { + endpoint: context.platform, + ...(options.input ? { lane: options.input } : {}), + domains, + values: context.scope?.values ?? {}, + now: context.now, + }; + const stories: DesktopStoryManifestEntry[] = []; + const decisions: WdioCaseDecision[] = []; + const selected = new Map>(); + const excluded: SelectionExclusions = { platform: 0, scope: 0, input: 0 }; + const legacy: string[] = []; + const entries = manifest.entries + .filter((entry) => entry.wdio && (!options.story || path.matchesGlob(entry.id, options.story))) + .filter((entry) => !options.tag || entry.tags.includes(options.tag)) + .sort((left, right) => left.id.localeCompare(right.id)); + for (const entry of entries) { + const cases = entry.wdio!.cases; + if (!cases?.length) { + if (!(entry.wdio!.testNames ?? ['default']).some((name) => matchesWdioTest(name, options.test))) continue; + if (options.input) legacy.push(entry.id); + else stories.push(entry); + continue; + } + const chosen = new Map(); + for (const { name, options: descriptor } of cases) { + if (!matchesWdioTest(name, options.test)) continue; + const outcome = decideDesktopCase(descriptor, target, `${entry.id} / ${name}`); + const decision: WdioCaseDecision = { storyId: entry.id, name, descriptor }; + if (outcome.kind === 'excluded') { + decision.excluded = outcome.reason; + excluded[outcome.reason]++; + } else { + if (outcome.kind === 'gap') decision.gap = outcome.gap; + chosen.set(name, decision); + } + decisions.push(decision); + } + if (chosen.size) { + stories.push(entry); + selected.set(entry.id, chosen); + } + } + if (legacy.length) { + const shown = legacy.slice(0, 10).join(', '); + const more = legacy.length > 10 ? `, and ${legacy.length - 10} more` : ''; + throw new Error(`Input lane "${options.input}" cannot run legacy wdio callbacks; declare them with desktopIt: ${shown}${more}.`); + } + return { stories, decisions, selected, excluded }; +} + +/** The work a selection announces before launch, so work that never starts is still accounted for. */ +export function plannedWdioWork( + selection: WdioSelection, + options: Pick, +): readonly { workId: string; label: string }[] { + return selection.stories.flatMap((entry) => { + const declared = selection.selected.get(entry.id); + const names = declared + ? [...declared.keys()] + : (entry.wdio?.testNames ?? ['default']).filter((name) => matchesWdioTest(name, options.test)); + return names.map((name) => ({ workId: csfTestId(entry.id, name), label: `${entry.id} / ${name}` })); + }); +} + +/** One declared case in a `--list` result. */ +export type WdioListedCase = Pick & { + name: string; + requires: NonNullable; + /** Selected but not executed: a declared `unsupported` entry or an active quarantine. */ + gap?: WdioDeclaredGap; +}; + +export type WdioListedStory = { + id: string; + sourcePath: string; + /** The names of a legacy callback, or the declared cases this target and lane run. */ + tests?: readonly string[] | readonly WdioListedCase[]; + /** Declared cases that matched the filters but do not run, each with the first reason it was excluded. */ + excluded?: readonly (WdioListedCase & { reason: keyof SelectionExclusions })[]; +}; + +/** What a selection would run, and why each other declared case that matched the filters would not. */ +export function listWdioSelection(selection: WdioSelection, options: Pick): WdioListedStory[] { + const decided = new Map(); + for (const decision of selection.decisions) { + const story = decided.get(decision.storyId); + if (story) story.push(decision); + else decided.set(decision.storyId, [decision]); + } + return selection.stories.map(({ id, sourcePath, wdio }) => { + const decisions = decided.get(id); + if (!selection.selected.has(id)) { + const names = wdio?.testNames?.filter((name) => matchesWdioTest(name, options.test)); + return { id, sourcePath, ...(names ? { tests: names } : {}) }; + } + const excluded = decisions!.flatMap((decision) => (decision.excluded ? [{ ...listedCase(decision), reason: decision.excluded }] : [])); + return { + id, + sourcePath, + tests: decisions!.filter(({ excluded }) => !excluded).map(listedCase), + ...(excluded.length ? { excluded } : {}), + }; + }); +} + +/** Names what a selection left out, so an empty selection explains itself. Empty when nothing was excluded. */ +export function describeWdioExclusions({ excluded }: Pick): string { + const { platform, scope, input } = excluded; + return platform + scope + input > 0 ? ` (excluded ${platform} by platform, ${scope} by scope, ${input} by input)` : ''; +} + +export function matchesWdioTest(name: string, pattern?: string): boolean { + return pattern === undefined || path.matchesGlob(name, pattern); +} + +export function csfTestId(storyId: string, testName?: string): string { + return `csf-${createHash('sha256') + .update(JSON.stringify([storyId, testName ?? 'default'])) + .digest('hex')}`; +} + +function listedCase({ name, descriptor, gap }: WdioCaseDecision): WdioListedCase { + const { input, inputSource, platforms, scope, requires = [] } = descriptor; + return { + name, + input, + inputSource, + requires, + ...(platforms ? { platforms } : {}), + ...(scope ? { scope } : {}), + ...(gap ? { gap } : {}), + }; +} diff --git a/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs b/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs index a8bae915fc..3736a9666e 100644 --- a/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs +++ b/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs @@ -25,6 +25,7 @@ async function main() { path.join(root, 'fixture.stories.ts'), ` import 'react-native-must-never-load-in-node'; + import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; export default { title: 'Fixture/Button' }; export const Pass = { tags: ['grouped', 'desktop-e2e'], @@ -88,6 +89,17 @@ async function main() { await expect(await browser.$('~button-primary')).toBeEnabled(); }, }}; + export const Lane = { tags: ['desktop-e2e'], wdio: [ + desktopIt('accessible only', { requires: ['accessibility-click'] }, async ({ browser, expect }) => { + await expect(await browser.$('~button-primary')).toBeEnabled(); + }), + desktopIt('either lane', async ({ browser, expect }) => { + await expect(await browser.$('~button-primary')).toBeEnabled(); + }), + desktopIt('physical only', { requires: ['physical-click'] }, async ({ browser, expect }) => { + await expect(await browser.$('~button-primary')).toBeEnabled(); + }), + ]}; `, ); const storyPackage = { name: 'inline-fixture', root, manifest: { name: 'inline-fixture' }, storyPatterns: ['*.stories.ts'] }; @@ -326,6 +338,46 @@ async function main() { } finally { await restrictedButtonHarness.close(); } + const laneResults = (results) => results.map(({ testName, status }) => ({ testName, status })); + // The accessible lane trusts the driver's own report, so a driver that does not show physical input disabled fails closed. + const unmaskedReport = new DesktopRunReporter({ + artifactsRoot: path.join(root, 'lane-reports'), + lane: { input: 'accessible', source: 'option', masking: 'host-masked', policyEnforced: false }, + summary: false, + }); + let unmaskedLane; + try { + await run('lane', { input: 'accessible', report: unmaskedReport }); + throw new Error('An accessible lane must not run against a driver that reports physical input enabled.'); + } catch (error) { + if (!(error instanceof Error) || !(error instanceof WdioStoryTestRunError) || !('results' in error) || !Array.isArray(error.results)) + throw error; + unmaskedLane = { results: laneResults(error.results), message: error.message }; + } + const unmaskedLaneResult = await unmaskedReport.finish(); + const maskedHarness = await createDesktopDriverStoryHarness(manifest, { + features: { keyboard: false, physicalClick: false, wheel: false }, + }); + let accessibleLane; + try { + accessibleLane = laneResults( + await run('lane', { input: 'accessible', targetId: maskedHarness.target.id, url: maskedHarness.server.url }), + ); + } finally { + await maskedHarness.close(); + } + const physicalLane = laneResults(await run('lane', { input: 'physical' })); + // Without a lane a declared physical case must not pin physical click: a host that masks it reports the case unsupported + // rather than failing to create the session, as the skip it replaces did not fail. + const unmaskedPhysicalHarness = await createDesktopDriverStoryHarness(manifest, { features: { physicalClick: false } }); + let compatibilityLane; + try { + compatibilityLane = laneResults( + await run('lane', { targetId: unmaskedPhysicalHarness.target.id, url: unmaskedPhysicalHarness.server.url }), + ); + } finally { + await unmaskedPhysicalHarness.close(); + } process.stdout.write( '\nWDIO_RESULT ' + JSON.stringify({ @@ -338,6 +390,18 @@ async function main() { smokeFailureLogged, button: button.map(({ testName, status }) => ({ testName, status })), restrictedButton: restrictedButton.map(({ testName, status }) => ({ testName, status })), + lane: { + accessible: accessibleLane, + physical: physicalLane, + compatibility: compatibilityLane, + unmasked: { + ...unmaskedLane, + status: unmaskedLaneResult.status, + failed: unmaskedLaneResult.counts.failed, + excluded: unmaskedLaneResult.selection?.excluded, + reported: unmaskedLaneResult.failures.map(({ error }) => error.message), + }, + }, platforms, namedFailureResults: namedFailureResults.map(({ testName, status }) => ({ testName, status })), namedTimeout, diff --git a/packages/agentic/storybook-desktop/src/testing/wdioRunner.test.ts b/packages/agentic/storybook-desktop/src/testing/wdioRunner.test.ts index 01a741bdfd..c69d9b8c56 100644 --- a/packages/agentic/storybook-desktop/src/testing/wdioRunner.test.ts +++ b/packages/agentic/storybook-desktop/src/testing/wdioRunner.test.ts @@ -23,6 +23,7 @@ test('executes isolated inline code and releases sessions after assertion failur !diagnostics.includes('wdio "fixture-button--fail" during execution') || !diagnostics.includes('wdio "fixture-button--navigation-failure" during navigation') || !diagnostics.includes('Storybook context did not settle.') || + !diagnostics.includes('wdio "fixture-button--lane / accessible only" during lane verification') || diagnostics.includes('Navigation refused before dispatch') || output.includes('[storybook] OK wdio "fixture-button--exit-zero"') ) { @@ -55,6 +56,30 @@ test('executes isolated inline code and releases sessions after assertion failur { testName: 'exposes enabled button semantics', status: 'passed' }, { testName: 'supports native pointer activation', status: 'skipped' }, ], + lane: { + accessible: [ + { testName: 'accessible only', status: 'passed' }, + { testName: 'either lane', status: 'passed' }, + ], + physical: [ + { testName: 'either lane', status: 'passed' }, + { testName: 'physical only', status: 'passed' }, + ], + // No lane, and a host that masks physical click: the physical case is unsupported, not a failed session. + compatibility: [ + { testName: 'accessible only', status: 'passed' }, + { testName: 'either lane', status: 'passed' }, + { testName: 'physical only', status: 'unsupported' }, + ], + // A driver that does not report physical input disabled stops the lane at its first case, reported once. + unmasked: { + results: [{ testName: 'accessible only', status: 'failed' }], + status: 'failed', + failed: 1, + excluded: { platform: 0, scope: 0, input: 1 }, + reported: [expect.stringContaining('driver --input accessible')], + }, + }, platforms: Object.fromEntries( ['macos', 'windows', 'win32'].map((platform) => [ platform, From 188114c0d2c526229e717126e8b1b1539c9a4012 Mon Sep 17 00:00:00 2001 From: Jason Morse Date: Fri, 6 Nov 2026 20:11:46 -0800 Subject: [PATCH 3/6] Migrate desktop story tests to declared input lanes Replace remaining component story runtime skips with desktopIt requirements, platform scopes and declared gaps. Carry the Win32 preview fixes from #4431 and split accessible and advisory physical Storybook CI lanes without changing aggregate PR requirements. Co-authored-by: Srividya Ramaswamy Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .changeset/components-desktop-it.md | 5 + .changeset/desktop-driver-ci-input-lanes.md | 5 + .changeset/storybook-runtime-win32-preview.md | 5 + .github/workflows/pr.yml | 52 +- apps/storybook/README.md | 6 +- apps/storybook/agent-map.yaml | 3 +- .../references/tests-and-stories.md | 59 +- .../src/common/desktopFocus.wdio.ts | 12 - .../src/components/avatar/avatar.stories.tsx | 46 +- .../src/components/button/button.stories.tsx | 641 +++++++++--------- .../src/components/card/card.stories.tsx | 13 +- .../components/checkbox/checkbox.stories.tsx | 35 +- .../components/dropdown/dropdown.stories.tsx | 238 +++---- .../src/components/input/input.stories.tsx | 134 ++-- .../listbox-item/listbox-item.stories.tsx | 144 ++-- .../menu-button/menu-button.stories.tsx | 108 +-- .../menu-item/menu-item.stories.tsx | 183 +++-- .../src/components/menu/menu.stories.tsx | 385 ++++++----- .../components/popover/popover.stories.tsx | 183 ++--- .../src/components/radio/radio.stories.tsx | 96 ++- .../components/skeleton/skeleton.stories.tsx | 57 +- .../components/spinner/spinner.stories.tsx | 49 +- .../src/components/switch/switch.stories.tsx | 59 +- .../components/tablist/tablist.stories.tsx | 147 ++-- .../src/components/tag/tag.stories.tsx | 17 +- .../focus-zone/focus-zone.stories.tsx | 114 ++-- .../layout-stable-text.stories.tsx | 9 +- .../references/ci-integration.md | 29 +- .../storybook-desktop-runtime/README.md | 5 + .../references/runtime-contract.md | 9 + .../src/StorybookUI.test.tsx | 83 +++ .../src/StorybookUI.win32.tsx | 10 +- .../createDesktopStorybookPreview.win32.tsx | 42 ++ .../driver/representativePlans.contract.cjs | 3 + .../src/testing/wdioRunner.test.ts | 3 +- 35 files changed, 1629 insertions(+), 1360 deletions(-) create mode 100644 .changeset/components-desktop-it.md create mode 100644 .changeset/desktop-driver-ci-input-lanes.md create mode 100644 .changeset/storybook-runtime-win32-preview.md create mode 100644 packages/agentic/storybook-desktop-runtime/src/StorybookUI.test.tsx create mode 100644 packages/agentic/storybook-desktop-runtime/src/createDesktopStorybookPreview.win32.tsx diff --git a/.changeset/components-desktop-it.md b/.changeset/components-desktop-it.md new file mode 100644 index 0000000000..a2dde1a2b6 --- /dev/null +++ b/.changeset/components-desktop-it.md @@ -0,0 +1,5 @@ +--- +'@fluentui-react-native/components': patch +--- + +Declare every component and primitive story test with `desktopIt`, so each case states the input it needs, the endpoints it runs on, and its known gaps instead of skipping at run time, and remove the internal `requireDesktopFocus` story helper. diff --git a/.changeset/desktop-driver-ci-input-lanes.md b/.changeset/desktop-driver-ci-input-lanes.md new file mode 100644 index 0000000000..b1984ac91f --- /dev/null +++ b/.changeset/desktop-driver-ci-input-lanes.md @@ -0,0 +1,5 @@ +--- +'@fluentui-react-native/desktop-driver': patch +--- + +Describe the Storybook CI input lanes: the hosted Windows and Win32 jobs run the accessible lane, and an advisory Win32 job runs the physical lane. diff --git a/.changeset/storybook-runtime-win32-preview.md b/.changeset/storybook-runtime-win32-preview.md new file mode 100644 index 0000000000..e83d056a3a --- /dev/null +++ b/.changeset/storybook-runtime-win32-preview.md @@ -0,0 +1,5 @@ +--- +'@fluentui-react-native/storybook-desktop-runtime': patch +--- + +Scroll the Win32 story preview inside its viewport and remount the preview for each navigation generation, so story tests start from a fresh, reachable preview. diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 6e324b297f..337071ba23 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -445,7 +445,7 @@ jobs: } - name: Smoke test Windows Storybook - run: yarn storybook smoke --windows --mode stories-and-tests + run: yarn storybook smoke --windows --mode stories-and-tests --input accessible working-directory: apps/storybook - name: Upload Windows Storybook artifacts @@ -510,7 +510,7 @@ jobs: if-no-files-found: warn win32-storybook: - name: Win32 Storybook PR + name: Win32 Storybook PR (accessible) runs-on: windows-latest timeout-minutes: 60 steps: @@ -537,14 +537,58 @@ jobs: disabled-input-features: keyboard,physicalClick,wheel - name: Smoke test Win32 Storybook - run: yarn storybook smoke --win32 --mode stories-and-tests + run: yarn storybook smoke --win32 --mode stories-and-tests --input accessible working-directory: apps/storybook - name: Upload Win32 Storybook artifacts if: ${{ always() }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: Storybook_win32_paper_${{ github.run_id }}_${{ github.run_attempt }} + name: Storybook_win32_paper_accessible_${{ github.run_id }}_${{ github.run_attempt }} + path: | + apps/storybook/artifacts/win32 + apps/storybook/artifacts/desktop + ${{ steps.desktop-driver.outputs.doctor-path }} + ${{ steps.desktop-driver.outputs.cache-root }}/v1/artifacts/**/build.log + ${{ steps.desktop-driver.outputs.cache-root }}/v1/diagnostics/** + if-no-files-found: warn + + # An advisory pilot outside the aggregate PR check: hosted physical input is not yet qualified (#4349). + win32-storybook-physical: + name: Win32 Storybook PR (physical) + runs-on: windows-latest + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up toolchain + uses: microsoft/react-native-test-app/.github/actions/setup-toolchain@1c99dd2947a77097bf937701e3dcc90aa109d17e # 5.4.15 + with: + platform: windows + node-version: 22 + + - name: Install dependencies + run: yarn + + - name: Build packages + run: yarn build + + - name: Set up Win32 Desktop Driver + id: desktop-driver + uses: ./.github/actions/setup-desktop-driver + with: + platform: win32 + + - name: Smoke test Win32 Storybook + run: yarn storybook smoke --win32 --mode stories-and-tests --input physical + working-directory: apps/storybook + + - name: Upload Win32 Storybook artifacts + if: ${{ always() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: Storybook_win32_paper_physical_${{ github.run_id }}_${{ github.run_attempt }} path: | apps/storybook/artifacts/win32 apps/storybook/artifacts/desktop diff --git a/apps/storybook/README.md b/apps/storybook/README.md index cbc89ee41d..50e75358cb 100644 --- a/apps/storybook/README.md +++ b/apps/storybook/README.md @@ -23,7 +23,11 @@ yarn storybook smoke --macos --mode stories-and-tests ``` Replace `macos` with `windows` or `win32`. Use `yarn storybook test --- --list` to discover executable story-test names. Runtime IDs and +-- --list` to discover executable story-test names. Add +`--input accessible` to `test` or `smoke --mode stories-and-tests` to exclude +cases that need physical input, or `--input physical` on an unattended +machine. Accessible runs avoid pointer and keyboard injection, but may still +activate windows or change focus; keep desktop workloads exclusive. Runtime IDs and listener ports are available through the app's `desktop-driver` command. See `agent-map.yaml` for the current command contract and paths. diff --git a/apps/storybook/agent-map.yaml b/apps/storybook/agent-map.yaml index b90f1bd39f..f8243cf7b1 100644 --- a/apps/storybook/agent-map.yaml +++ b/apps/storybook/agent-map.yaml @@ -31,11 +31,12 @@ services: executable_tests: command: yarn storybook test -- discovery: yarn storybook test -- --list - authoring: top-level wdio callbacks or named callback collections in CSF3 stories + authoring: desktopIt cases with input, platform, and gap descriptors in CSF3 story wdio arrays config: desktop.config.mts Storybook integration wdio settings lifecycle: attached-session-per-test-with-isolated-node-worker context_platform: macos-or-windows-or-win32-target-endpoint test_filter: --test + input_lane: --input accessible|physical smoke_mode: stories-and-tests execution: navigate-and-remount-then-run-tests-grouped-by-story diff --git a/packages/agentic/components/references/tests-and-stories.md b/packages/agentic/components/references/tests-and-stories.md index edc3276894..5691b7398f 100644 --- a/packages/agentic/components/references/tests-and-stories.md +++ b/packages/agentic/components/references/tests-and-stories.md @@ -137,23 +137,41 @@ validate. Prefer executable WebdriverIO tests under `wdio`, typed with `WdioStory>` from `@fluentui-react-native/storybook-desktop/testing` using `import type`. -The story is the suite (`describe`), and named functions are its cases (`it`): +The story is the suite (`describe`). Declare each case (`it`) with `desktopIt` +from the same entry point, adding a descriptor when the case needs host input +or applies only to some endpoints: ```tsx export const Default: Story = { - wdio: { - 'is enabled': async ({ browser, expect }) => { + wdio: [ + desktopIt('is enabled', async ({ browser, expect }) => { await expect(await browser.$('~story-button')).toBeEnabled(); - }, - }, + }), + desktopIt('moves focus with Tab', { requires: ['physical-click', 'keyboard', 'focus'] }, async ({ browser, expect }) => { + // ... + }), + ], }; ``` Each callback receives `browser`, `expect`, `platform`, `desktop`, `signal`, and `skip`. The typed target `platform` is `macos`, `windows`, or `win32`; Win32 is distinct from its WebDriver `platformName` of `windows`. Branch -inside callbacks for genuine platform differences and use explicit skip -reasons for unsupported `browser.capabilities['furn:features']`. +inside callbacks for genuine platform differences in one assertion. Do not +call `skip` or feature-check `browser.capabilities['furn:features']` for a +condition the descriptor can state: + +- `requires` lists the host capabilities the case uses, and its physical + entries put the case in the physical input lane. Focus and keyboard cases + declare `['physical-click', 'keyboard', 'focus']`. A case that reads only + native state declares nothing and runs in both lanes. Use `click` when either + click transport proves the behavior, `physical-click` for pointer-specific + behavior, and `accessibility-click` for UIA or AX action dispatch. +- `platforms` scopes the case to endpoints, such as `['win32']` or `['!macos']`. + Split a case when endpoints need different transports rather than branching + on the click mode. +- `unsupported` records a known product gap on named endpoints with a reason, + and `quarantine` suspends a flaky case with an issue and an `expires` date. All catalog tests use WDIO; custom `desktopDriver` plans remain only in runner compatibility fixtures. Keep names static and callbacks self-contained. Node helpers may be @@ -162,11 +180,13 @@ story-module bindings. The shared Babel config strips all test functions before native dependency collection. Named cases get independent previews, sessions, workers, deadlines, results, and failure evidence. -The single-function `wdio` form remains supported. Run with -`yarn storybook test --` and optionally `--test `. -`--list` discovers names without executing code. The `stories-and-tests` +Run with `yarn storybook test --`, optionally with `--test ` +and `--input accessible|physical` to select one input lane. `--list` discovers +cases and their resolved input without executing code. The `stories-and-tests` smoke mode runs these functions alongside remaining legacy plans, grouped by -story. See the executable test contract (`@fluentui-react-native/storybook-desktop/references/story-tests.md`). +story. See the executable test contract +(`@fluentui-react-native/storybook-desktop/references/story-tests.md`) for the +full descriptor, lane, and reporting rules. The components package's `test:stories` project checks every test-bearing story without emitting it into the component library. Add new test-bearing files to @@ -191,9 +211,12 @@ timing. ### Desktop focus test scope Keep shared requirements executable on Windows Fabric, Office Win32, and macOS. -`requireDesktopFocus` checks native capabilities, not an operating-system -allowlist. A missing capability skips the case with a reason; a platform -difference usually changes one assertion rather than skipping the entire case. +Focus cases declare `requires: ['physical-click', 'keyboard', 'focus']`, which +describes native capabilities, not an operating-system allowlist. The physical +lane runs them, the accessible lane excludes them, and a run without a lane +reports a case whose capability the host lacks as unsupported, with its reason, +instead of executing it. A platform difference +usually changes one assertion rather than excluding the entire case. | Behavior | Scope and expected result | | -------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -225,13 +248,15 @@ resolved-style coverage without incidental scene rerenders, plus native visual evidence for renderer appearance. Likewise, a checked/selected state read after a physical click does not exercise UIA/AX action dispatch. -Report passed, failed, and skipped counts together. A zero-exit smoke traversal -with platform-skipped focus cases is not cross-platform focus qualification. +Report passed, failed, skipped, unsupported, and excluded counts together. A +zero-exit accessible-lane run excludes every focus case and is not +cross-platform focus qualification. The macOS Fabric View does not project disabled or selected accessibility state to AXEnabled/AXSelected (C-A11Y-005 in the repository evidence reference `accessibility.action-transport-evidence`). Those native -announcement assertions are isolated in explicitly skipped cases; shared +announcement assertions are isolated in cases declaring +`unsupported: { platforms: ['macos'], reason }`; shared focus, activation, and selection-callback cases still run. Do not substitute rendered state for an AX result or describe these skips as intended macOS semantics. Track the native projection gap and revisit the cases when the diff --git a/packages/agentic/components/src/common/desktopFocus.wdio.ts b/packages/agentic/components/src/common/desktopFocus.wdio.ts index c22e292b55..e2432891fd 100644 --- a/packages/agentic/components/src/common/desktopFocus.wdio.ts +++ b/packages/agentic/components/src/common/desktopFocus.wdio.ts @@ -1,17 +1,5 @@ import type { WdioStoryContext } from '@fluentui-react-native/storybook-desktop/testing'; -export function requireDesktopFocus({ browser, skip }: Pick): boolean { - const features = browser.capabilities['furn:features']; - if (!features) { - throw new Error('Desktop Driver did not provide feature capabilities.'); - } - if (!features.physicalClick || !features.keyboard || !features.focus) { - skip('Native focus qualification requires physical pointer input, keyboard input, and focus observation.'); - return false; - } - return true; -} - /** Click-to-focus is Windows-specific; ordinary AppKit controls need not take focus on click. */ export async function expectWindowsPointerFocus( { browser, platform }: Pick, diff --git a/packages/agentic/components/src/components/avatar/avatar.stories.tsx b/packages/agentic/components/src/components/avatar/avatar.stories.tsx index f27248cc3d..2f1c700e57 100644 --- a/packages/agentic/components/src/components/avatar/avatar.stories.tsx +++ b/packages/agentic/components/src/components/avatar/avatar.stories.tsx @@ -3,6 +3,7 @@ import type { ReactNode } from 'react'; import { StyleSheet, Text, View } from 'react-native'; import type { Meta, StoryObj } from '@storybook/react-native'; +import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; import type { WdioStory } from '@fluentui-react-native/storybook-desktop/testing'; import { Avatar } from './avatar'; @@ -108,32 +109,35 @@ export const ActivityRing: Story = { }, }, }, - wdio: { - 'AVT-004 exposes the informative identity as a named native image': async ({ browser, expect }) => { + wdio: [ + desktopIt('AVT-004 exposes the informative identity as a named native image', async ({ browser, expect }) => { const avatar = await browser.$('~agentic-storybook-avatar-activity-ring'); await expect(avatar).toExist(); expect(await avatar.getTagName()).toBe('image'); expect(await avatar.getProperty('name')).toBe('Lydia Mitchelson'); - }, - 'toggles the activity ring at every size without crashing or changing avatar bounds': async ({ browser, desktop, expect }) => { - const assert: typeof import('node:assert') = (await import('node:assert')).default; - for (const size of [16, 20, 24, 28, 32, 40, 56, 120]) { - for (const activityRing of [false, true]) { - await desktop.session.updateStoryArgs('components-avatar--activity-ring', { activityRing, size }); - await expect(await browser.$('~agentic-storybook-avatar-ring-state')).toHaveText( - `Activity ring ${activityRing ? 'on' : 'off'}, size ${size}`, - ); - const avatar = await browser.$('~agentic-storybook-avatar-activity-ring'); - await expect(avatar).toExist(); - const bounds = await browser.getElementRect(await avatar.elementId); - assert( - Math.abs(bounds.width - size) <= 1 && Math.abs(bounds.height - size) <= 1, - 'The activity ring must not resize the avatar.', - ); + }), + desktopIt( + 'toggles the activity ring at every size without crashing or changing avatar bounds', + async ({ browser, desktop, expect }) => { + const assert: typeof import('node:assert') = (await import('node:assert')).default; + for (const size of [16, 20, 24, 28, 32, 40, 56, 120]) { + for (const activityRing of [false, true]) { + await desktop.session.updateStoryArgs('components-avatar--activity-ring', { activityRing, size }); + await expect(await browser.$('~agentic-storybook-avatar-ring-state')).toHaveText( + `Activity ring ${activityRing ? 'on' : 'off'}, size ${size}`, + ); + const avatar = await browser.$('~agentic-storybook-avatar-activity-ring'); + await expect(avatar).toExist(); + const bounds = await browser.getElementRect(await avatar.elementId); + assert( + Math.abs(bounds.width - size) <= 1 && Math.abs(bounds.height - size) <= 1, + 'The activity ring must not resize the avatar.', + ); + } } - } - }, - }, + }, + ), + ], }; const styles = StyleSheet.create({ diff --git a/packages/agentic/components/src/components/button/button.stories.tsx b/packages/agentic/components/src/components/button/button.stories.tsx index a178b57d9a..b20507a710 100644 --- a/packages/agentic/components/src/components/button/button.stories.tsx +++ b/packages/agentic/components/src/components/button/button.stories.tsx @@ -8,6 +8,7 @@ import type { Meta, StoryObj } from '@storybook/react-native'; import { ThemedRoot, useRootInputModality } from '@fluentui-react-native/design'; import { resolveAccessibilityAction } from '@fluentui-react-native/framework-base'; import type { FocusKeyboardEvent, FocusRequest } from '@fluentui-react-native/framework-base'; +import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; import type { WdioStory } from '@fluentui-react-native/storybook-desktop/testing'; import { Button } from './button'; @@ -96,28 +97,26 @@ type Story = WdioStory>; export const Default: Story = { tags: ['desktop-e2e'], - wdio: { - 'exposes enabled button semantics': async ({ browser, expect }) => { + wdio: [ + desktopIt('exposes enabled button semantics', async ({ browser, expect }) => { const assert: typeof import('node:assert') = (await import('node:assert')).default; const button = await browser.$('~agentic-storybook-button'); await expect(button).toExist(); await expect(button).toBeEnabled(); assert.strictEqual(await button.getTagName(), 'button'); - }, - 'supports native pointer activation': async ({ browser, expect, platform, skip }) => { - const { expectWindowsPointerFocus } = await import('../../common/desktopFocus.wdio.ts'); - const features = browser.capabilities['furn:features']; - if (!features) throw new Error('Desktop Driver did not provide feature capabilities.'); - if (!features.physicalClick || !features.elementScreenshot || (platform !== 'macos' && !features.focus)) { - skip('This test requires physical clicks, element screenshots, and focus on Windows/Win32.'); - return; - } - const button = await browser.$('~agentic-storybook-button'); - await button.click(); - await expectWindowsPointerFocus({ browser, platform }, 'agentic-storybook-button'); - expect(await browser.takeElementScreenshot(await button.elementId)).toMatch(/^iVBORw0KGgo/); - }, - }, + }), + desktopIt( + 'supports native pointer activation', + { requires: ['physical-click', 'element-screenshot', 'focus'] }, + async ({ browser, expect, platform }) => { + const { expectWindowsPointerFocus } = await import('../../common/desktopFocus.wdio.ts'); + const button = await browser.$('~agentic-storybook-button'); + await button.click(); + await expectWindowsPointerFocus({ browser, platform }, 'agentic-storybook-button'); + expect(await browser.takeElementScreenshot(await button.elementId)).toMatch(/^iVBORw0KGgo/); + }, + ), + ], }; function ModalityProbe({ testID = 'focus-input-modality' }: { testID?: string }) { @@ -168,149 +167,165 @@ function FocusManagementScene() { export const FocusManagement: Story = { render: () => , tags: ['desktop-focus'], - wdio: { - 'honors pointer focus policy and invokes exactly once per Enter and Space': async (context) => { - const { requireDesktopFocus, expectNativeState, expectWindowsPointerFocus, focusByTab } = - await import('../../common/desktopFocus.wdio.ts'); - if (!requireDesktopFocus(context)) return; - const { browser, expect } = context; - const button = await browser.$('~focus-probe-button'); - const count = await browser.$('~focus-probe-count'); - await (await browser.$('~focus-probe-entry')).click(); - await expectNativeState(browser, 'focus-probe-entry', 'focused', true); - await button.click(); - await expectWindowsPointerFocus(context, 'focus-probe-button'); - await expect(count).toHaveText('1'); - await focusByTab(browser, 'focus-probe-entry', 'focus-probe-button'); - await expect(await browser.$('~focus-probe-state')).toHaveText('focused'); - await browser.keys('\uE007'); - await expect(count).toHaveText('2'); - await expect(await browser.$('~focus-input-modality')).toHaveText('keyboard'); - await browser.keys('\uE00D'); - await expect(count).toHaveText('3'); - await button.click(); - await expect(count).toHaveText('4'); - await expect(await browser.$('~focus-input-modality')).toHaveText('pointer'); - }, - 'updates same-target modality without refocusing and shares it with nested themes': async (context) => { - const { requireDesktopFocus, expectNativeState, expectWindowsPointerFocus, focusByTab } = - await import('../../common/desktopFocus.wdio.ts'); - if (!requireDesktopFocus(context)) return; - const { browser, expect } = context; - const button = await browser.$('~focus-probe-button'); - await focusByTab(browser, 'focus-probe-entry', 'focus-probe-button'); - await button.click(); - await expect(await browser.$('~focus-input-modality')).toHaveText('pointer'); - await expect(await browser.$('~focus-probe-focus-count')).toHaveText('1'); - await browser.keys('x'); - await expect(await browser.$('~focus-input-modality')).toHaveText('keyboard'); - await expect(await browser.$('~focus-nested-modality')).toHaveText('keyboard'); - await expect(await browser.$('~focus-probe-count')).toHaveText('1'); - await button.click(); - await expect(await browser.$('~focus-input-modality')).toHaveText('pointer'); - await expect(await browser.$('~focus-probe-focus-count')).toHaveText('1'); - await (await browser.$('~focus-probe-nested')).click(); - await expectWindowsPointerFocus(context, 'focus-probe-nested'); - await focusByTab(browser, 'focus-probe-entry', 'focus-probe-button'); - await browser.keys('\uE004'); - await expectNativeState(browser, 'focus-probe-next', 'focused', true); - await browser.keys('\uE004'); - await expectNativeState(browser, 'focus-probe-nested', 'focused', true); - await browser.keys('x'); - await expect(await browser.$('~focus-input-modality')).toHaveText('keyboard'); - await expect(await browser.$('~focus-nested-modality')).toHaveText('keyboard'); - await expect(await browser.$('~focus-probe-state')).toHaveText('blurred'); - }, - 'skips disabled stops and does not activate the new owner on a stale key-up': async (context) => { - const { requireDesktopFocus, expectNativeState, focusByTab } = await import('../../common/desktopFocus.wdio.ts'); - if (!requireDesktopFocus(context)) return; - const { browser, expect, platform } = context; - await focusByTab(browser, 'focus-probe-entry', 'focus-probe-button'); - try { - await browser.performActions([ - { - type: 'key', - id: 'focus-keyboard', - actions: [ - { type: 'keyDown', value: '\uE007' }, - { type: 'keyDown', value: '\uE004' }, - { type: 'keyUp', value: '\uE004' }, - { type: 'keyUp', value: '\uE007' }, - ], - }, - ]); - } finally { - await browser.releaseActions(); - } - await expectNativeState(browser, 'focus-probe-next', 'focused', true); - await expectNativeState(browser, 'focus-probe-disabled', 'focused', false); - await expect(await browser.$('~focus-probe-state')).toHaveText('blurred'); - await expect(await browser.$('~focus-probe-count')).toHaveText(platform === 'macos' ? '1' : '0'); - await expect(await browser.$('~focus-probe-pressed')).toHaveText('false'); - await expect(await browser.$('~focus-probe-next-count')).toHaveText('0'); - await browser.keys(['\uE008', '\uE004']); - await expectNativeState(browser, 'focus-probe-button', 'focused', true); - }, - 'suppresses repeats and activates at the native platform key phase': async (context) => { - const { requireDesktopFocus, focusByTab } = await import('../../common/desktopFocus.wdio.ts'); - if (!requireDesktopFocus(context)) return; - const { browser, expect, platform } = context; - await focusByTab(browser, 'focus-probe-entry', 'focus-probe-button'); - try { - await browser.performActions([ - { - type: 'key', - id: 'focus-keyboard', - actions: [ - { type: 'keyDown', value: '\uE00D' }, - { type: 'keyDown', value: '\uE00D' }, - { type: 'keyDown', value: '\uE00D' }, - ], - }, - ]); - await expect(await browser.$('~focus-probe-count')).toHaveText(platform === 'macos' ? '1' : '0'); - await browser.performActions([{ type: 'key', id: 'focus-keyboard', actions: [{ type: 'keyUp', value: '\uE00D' }] }]); + wdio: [ + desktopIt( + 'honors pointer focus policy and invokes exactly once per Enter and Space', + { requires: ['physical-click', 'keyboard', 'focus'] }, + async (context) => { + const { expectNativeState, expectWindowsPointerFocus, focusByTab } = await import('../../common/desktopFocus.wdio.ts'); + const { browser, expect } = context; + const button = await browser.$('~focus-probe-button'); + const count = await browser.$('~focus-probe-count'); + await (await browser.$('~focus-probe-entry')).click(); + await expectNativeState(browser, 'focus-probe-entry', 'focused', true); + await button.click(); + await expectWindowsPointerFocus(context, 'focus-probe-button'); + await expect(count).toHaveText('1'); + await focusByTab(browser, 'focus-probe-entry', 'focus-probe-button'); + await expect(await browser.$('~focus-probe-state')).toHaveText('focused'); + await browser.keys('\uE007'); + await expect(count).toHaveText('2'); + await expect(await browser.$('~focus-input-modality')).toHaveText('keyboard'); + await browser.keys('\uE00D'); + await expect(count).toHaveText('3'); + await button.click(); + await expect(count).toHaveText('4'); + await expect(await browser.$('~focus-input-modality')).toHaveText('pointer'); + }, + ), + desktopIt( + 'updates same-target modality without refocusing and shares it with nested themes', + { requires: ['physical-click', 'keyboard', 'focus'] }, + async (context) => { + const { expectNativeState, expectWindowsPointerFocus, focusByTab } = await import('../../common/desktopFocus.wdio.ts'); + const { browser, expect } = context; + const button = await browser.$('~focus-probe-button'); + await focusByTab(browser, 'focus-probe-entry', 'focus-probe-button'); + await button.click(); + await expect(await browser.$('~focus-input-modality')).toHaveText('pointer'); + await expect(await browser.$('~focus-probe-focus-count')).toHaveText('1'); + await browser.keys('x'); + await expect(await browser.$('~focus-input-modality')).toHaveText('keyboard'); + await expect(await browser.$('~focus-nested-modality')).toHaveText('keyboard'); await expect(await browser.$('~focus-probe-count')).toHaveText('1'); - await expect(await browser.$('~focus-probe-pressed')).toHaveText('false'); - } finally { - await browser.releaseActions(); - } - }, - 'releases pressed feedback after an intervening key without a stale activation': async (context) => { - if (context.platform === 'win32') { - context.skip('Quarantined until #4373 is fixed: Win32 does not forward onPressOut after Space with an intervening key.'); - return; - } - const { requireDesktopFocus, focusByTab } = await import('../../common/desktopFocus.wdio.ts'); - if (!requireDesktopFocus(context)) return; - const { browser, expect, platform } = context; - await focusByTab(browser, 'focus-probe-entry', 'focus-probe-button'); - try { - await browser.performActions([ - { - type: 'key', - id: 'focus-keyboard', - actions: [ - { type: 'keyDown', value: '\uE00D' }, - { type: 'keyDown', value: 'x' }, - { type: 'keyUp', value: 'x' }, - { type: 'keyUp', value: '\uE00D' }, - ], - }, - ]); + await button.click(); + await expect(await browser.$('~focus-input-modality')).toHaveText('pointer'); + await expect(await browser.$('~focus-probe-focus-count')).toHaveText('1'); + await (await browser.$('~focus-probe-nested')).click(); + await expectWindowsPointerFocus(context, 'focus-probe-nested'); + await focusByTab(browser, 'focus-probe-entry', 'focus-probe-button'); + await browser.keys('\uE004'); + await expectNativeState(browser, 'focus-probe-next', 'focused', true); + await browser.keys('\uE004'); + await expectNativeState(browser, 'focus-probe-nested', 'focused', true); + await browser.keys('x'); + await expect(await browser.$('~focus-input-modality')).toHaveText('keyboard'); + await expect(await browser.$('~focus-nested-modality')).toHaveText('keyboard'); + await expect(await browser.$('~focus-probe-state')).toHaveText('blurred'); + }, + ), + desktopIt( + 'skips disabled stops and does not activate the new owner on a stale key-up', + { requires: ['physical-click', 'keyboard', 'focus'] }, + async (context) => { + const { expectNativeState, focusByTab } = await import('../../common/desktopFocus.wdio.ts'); + const { browser, expect, platform } = context; + await focusByTab(browser, 'focus-probe-entry', 'focus-probe-button'); + try { + await browser.performActions([ + { + type: 'key', + id: 'focus-keyboard', + actions: [ + { type: 'keyDown', value: '\uE007' }, + { type: 'keyDown', value: '\uE004' }, + { type: 'keyUp', value: '\uE004' }, + { type: 'keyUp', value: '\uE007' }, + ], + }, + ]); + } finally { + await browser.releaseActions(); + } + await expectNativeState(browser, 'focus-probe-next', 'focused', true); + await expectNativeState(browser, 'focus-probe-disabled', 'focused', false); + await expect(await browser.$('~focus-probe-state')).toHaveText('blurred'); await expect(await browser.$('~focus-probe-count')).toHaveText(platform === 'macos' ? '1' : '0'); await expect(await browser.$('~focus-probe-pressed')).toHaveText('false'); - await expect(await browser.$('~focus-probe-release-count')).toHaveText('1'); - } finally { - await browser.releaseActions(); - } - }, - 'exposes the native disabled accessibility state': async ({ browser, expect }) => { + await expect(await browser.$('~focus-probe-next-count')).toHaveText('0'); + await browser.keys(['\uE008', '\uE004']); + await expectNativeState(browser, 'focus-probe-button', 'focused', true); + }, + ), + desktopIt( + 'suppresses repeats and activates at the native platform key phase', + { requires: ['physical-click', 'keyboard', 'focus'] }, + async (context) => { + const { focusByTab } = await import('../../common/desktopFocus.wdio.ts'); + const { browser, expect, platform } = context; + await focusByTab(browser, 'focus-probe-entry', 'focus-probe-button'); + try { + await browser.performActions([ + { + type: 'key', + id: 'focus-keyboard', + actions: [ + { type: 'keyDown', value: '\uE00D' }, + { type: 'keyDown', value: '\uE00D' }, + { type: 'keyDown', value: '\uE00D' }, + ], + }, + ]); + await expect(await browser.$('~focus-probe-count')).toHaveText(platform === 'macos' ? '1' : '0'); + await browser.performActions([{ type: 'key', id: 'focus-keyboard', actions: [{ type: 'keyUp', value: '\uE00D' }] }]); + await expect(await browser.$('~focus-probe-count')).toHaveText('1'); + await expect(await browser.$('~focus-probe-pressed')).toHaveText('false'); + } finally { + await browser.releaseActions(); + } + }, + ), + desktopIt( + 'releases pressed feedback after an intervening key without a stale activation', + { + requires: ['physical-click', 'keyboard', 'focus'], + quarantine: { + issue: '#4373', + reason: 'Win32 does not forward onPressOut after Space with an intervening key.', + expires: '2027-01-31', + platforms: ['win32'], + }, + }, + async (context) => { + const { focusByTab } = await import('../../common/desktopFocus.wdio.ts'); + const { browser, expect, platform } = context; + await focusByTab(browser, 'focus-probe-entry', 'focus-probe-button'); + try { + await browser.performActions([ + { + type: 'key', + id: 'focus-keyboard', + actions: [ + { type: 'keyDown', value: '\uE00D' }, + { type: 'keyDown', value: 'x' }, + { type: 'keyUp', value: 'x' }, + { type: 'keyUp', value: '\uE00D' }, + ], + }, + ]); + await expect(await browser.$('~focus-probe-count')).toHaveText(platform === 'macos' ? '1' : '0'); + await expect(await browser.$('~focus-probe-pressed')).toHaveText('false'); + await expect(await browser.$('~focus-probe-release-count')).toHaveText('1'); + } finally { + await browser.releaseActions(); + } + }, + ), + desktopIt('exposes the native disabled accessibility state', async ({ browser, expect }) => { await expect(await browser.$('~focus-probe-disabled')).not.toBeEnabled(); - }, - 'releases repeated modifiers before the next Tab': async (context) => { - const { requireDesktopFocus, expectNativeState, focusByTab } = await import('../../common/desktopFocus.wdio.ts'); - if (!requireDesktopFocus(context)) return; + }), + desktopIt('releases repeated modifiers before the next Tab', { requires: ['physical-click', 'keyboard', 'focus'] }, async (context) => { + const { expectNativeState, focusByTab } = await import('../../common/desktopFocus.wdio.ts'); const { browser } = context; await focusByTab(browser, 'focus-probe-entry', 'focus-probe-button'); try { @@ -331,8 +346,8 @@ export const FocusManagement: Story = { } finally { await browser.releaseActions(); } - }, - }, + }), + ], }; function FocusRequestsScene() { @@ -361,32 +376,35 @@ function FocusRequestsScene() { export const FocusRequests: Story = { render: () => , tags: ['desktop-focus'], - wdio: { - 'confirms native requests without changing modality and rejects disabled or detached targets': async (context) => { - const { requireDesktopFocus, expectNativeState } = await import('../../common/desktopFocus.wdio.ts'); - if (!requireDesktopFocus(context)) return; - const { browser, expect, platform } = context; - const command = await browser.$('~focus-request-command'); - const status = await browser.$('~focus-request-status'); - await command.click(); - await expectNativeState(browser, 'focus-request-target', 'focused', true); - await expect(status).toHaveText('confirmed'); - await expect(await browser.$('~focus-input-modality')).toHaveText('pointer'); - await (await browser.$('~focus-request-entry')).click(); - await expectNativeState(browser, 'focus-request-entry', 'focused', true); - await (await browser.$('~focus-request-disable')).click(); - await expectNativeState(browser, 'focus-request-target', 'focused', false); - if (platform !== 'macos') { - await expect(await browser.$('~focus-request-target')).not.toBeEnabled(); - } - await command.click(); - await expect(status).toHaveText('not-focusable'); - await (await browser.$('~focus-request-mount')).click(); - await expect(await browser.$('~focus-request-target')).not.toExist(); - await command.click(); - await expect(status).toHaveText('not-mounted'); - }, - }, + wdio: [ + desktopIt( + 'confirms native requests without changing modality and rejects disabled or detached targets', + { requires: ['physical-click', 'keyboard', 'focus'] }, + async (context) => { + const { expectNativeState } = await import('../../common/desktopFocus.wdio.ts'); + const { browser, expect, platform } = context; + const command = await browser.$('~focus-request-command'); + const status = await browser.$('~focus-request-status'); + await command.click(); + await expectNativeState(browser, 'focus-request-target', 'focused', true); + await expect(status).toHaveText('confirmed'); + await expect(await browser.$('~focus-input-modality')).toHaveText('pointer'); + await (await browser.$('~focus-request-entry')).click(); + await expectNativeState(browser, 'focus-request-entry', 'focused', true); + await (await browser.$('~focus-request-disable')).click(); + await expectNativeState(browser, 'focus-request-target', 'focused', false); + if (platform !== 'macos') { + await expect(await browser.$('~focus-request-target')).not.toBeEnabled(); + } + await command.click(); + await expect(status).toHaveText('not-focusable'); + await (await browser.$('~focus-request-mount')).click(); + await expect(await browser.$('~focus-request-target')).not.toExist(); + await command.click(); + await expect(status).toHaveText('not-mounted'); + }, + ), + ], }; export const Overview: Story = { @@ -518,41 +536,35 @@ export const Selected: Story = { }; export const ExternallyDrivenSelection: Story = { - wdio: { - 'starts unselected': async ({ browser, expect }) => { + wdio: [ + desktopIt('starts unselected', async ({ browser, expect }) => { await expect(await browser.$('~agentic-storybook-button-selection-state')).toHaveText('Not selected'); - }, - 'updates and resets caller-owned selection': async ({ browser, expect, platform, skip }) => { - const features = browser.capabilities['furn:features']; - if (!features) throw new Error('Desktop Driver did not provide feature capabilities.'); - if (platform === 'macos' && !features.physicalClick) { - skip('macOS caller-owned selection requires physical pointer input; the custom Toggle action is not an AXPress route.'); - return; - } - if (!features.physicalClick && !features.accessibilityClick) { - skip('This test requires native accessibility activation or physical pointer input.'); - return; - } - const { clickPhysical } = await import('../../common/desktopFocus.wdio.ts'); + }), + desktopIt('updates and resets caller-owned selection', { requires: ['click'], platforms: ['!macos'] }, async ({ browser, expect }) => { const favorite = await browser.$('~agentic-storybook-button-favorite'); - const reset = await browser.$('~agentic-storybook-button-reset'); const status = await browser.$('~agentic-storybook-button-selection-state'); await expect(favorite).toBeEnabled(); await expect(status).toHaveText('Not selected'); - if (platform === 'macos') { - await clickPhysical(browser, 'agentic-storybook-button-favorite'); - } else { - await favorite.click(); - } + await favorite.click(); await expect(status).toHaveText('Selected'); - if (platform === 'macos') { - await clickPhysical(browser, 'agentic-storybook-button-reset'); - } else { - await reset.click(); - } + await (await browser.$('~agentic-storybook-button-reset')).click(); await expect(status).toHaveText('Not selected'); - }, - }, + }), + desktopIt( + 'updates and resets caller-owned selection with a physical pointer', + { requires: ['physical-click'], platforms: ['macos'] }, + async ({ browser, expect }) => { + const { clickPhysical } = await import('../../common/desktopFocus.wdio.ts'); + const status = await browser.$('~agentic-storybook-button-selection-state'); + await expect(await browser.$('~agentic-storybook-button-favorite')).toBeEnabled(); + await expect(status).toHaveText('Not selected'); + await clickPhysical(browser, 'agentic-storybook-button-favorite'); + await expect(status).toHaveText('Selected'); + await clickPhysical(browser, 'agentic-storybook-button-reset'); + await expect(status).toHaveText('Not selected'); + }, + ), + ], render: () => { const ToggleGroup = () => { const [selected, setSelected] = useState(false); @@ -624,8 +636,8 @@ export const NativeNames: Story = {