From f5a90cdda9b429b72d9ef9b1429211a114ef71ab Mon Sep 17 00:00:00 2001 From: Jason Morse Date: Fri, 6 Nov 2026 14:47:43 -0800 Subject: [PATCH 1/2] WDIO descriptor and desktopIt helper --- .changeset/input-lanes.md | 6 + .../desktop-driver/src/authoring/index.ts | 9 + .../src/authoring/input.test.ts | 32 +++ .../desktop-driver/src/authoring/input.ts | 167 +++++++++++++++ .../hosts/native/NativeDesktopHost.test.ts | 14 ++ .../src/hosts/native/NativeDesktopHost.ts | 12 +- packages/agentic/desktop-driver/src/index.ts | 9 + .../agentic/desktop-driver/src/storybook.ts | 8 +- .../config/transform-story-tests.cjs | 83 ++++++++ .../config/transform-story-tests.d.cts | 3 +- .../storybook-desktop/references/commands.md | 8 + .../references/story-tests.md | 45 +++- .../src/cli/DesktopStorybookCli.ts | 96 +++++++-- .../src/cli/createDesktopStorybookCommand.ts | 17 +- .../storybook-desktop/src/cli/smokeTests.ts | 4 +- .../storybook-desktop/src/config/commands.ts | 1 + .../src/config/transformStoryTests.test.ts | 10 + .../storybook-desktop/src/config/wdio.test.ts | 16 ++ .../storybook-desktop/src/config/wdio.ts | 23 +- .../src/driver/driverManifest.ts | 4 +- .../src/driver/storyManifest.ts | 12 +- .../src/testing/desktopIt.ts | 11 + .../src/testing/extractWdioTests.test.ts | 31 +++ .../src/testing/extractWdioTests.ts | 7 +- .../storybook-desktop/src/testing/index.ts | 3 +- .../src/testing/runWdioTests.test.ts | 68 ++++++ .../src/testing/runWdioTests.ts | 197 +++++++++++++++--- .../storybook-desktop/src/testing/types.ts | 5 +- .../src/testing/wdio.types.test.ts | 14 +- 29 files changed, 844 insertions(+), 71 deletions(-) create mode 100644 .changeset/input-lanes.md create mode 100644 packages/agentic/desktop-driver/src/authoring/input.test.ts create mode 100644 packages/agentic/desktop-driver/src/authoring/input.ts create mode 100644 packages/agentic/storybook-desktop/src/testing/desktopIt.ts create mode 100644 packages/agentic/storybook-desktop/src/testing/runWdioTests.test.ts diff --git a/.changeset/input-lanes.md b/.changeset/input-lanes.md new file mode 100644 index 0000000000..4c06b6b268 --- /dev/null +++ b/.changeset/input-lanes.md @@ -0,0 +1,6 @@ +--- +'@fluentui-react-native/desktop-driver': patch +'@fluentui-react-native/storybook-desktop': patch +--- + +Add modality-aware desktop test descriptors and Storybook input-lane selection. diff --git a/packages/agentic/desktop-driver/src/authoring/index.ts b/packages/agentic/desktop-driver/src/authoring/index.ts index abc515c0d9..0fd5cd6695 100644 --- a/packages/agentic/desktop-driver/src/authoring/index.ts +++ b/packages/agentic/desktop-driver/src/authoring/index.ts @@ -14,6 +14,15 @@ export { desktopStoryPlatforms, validateDesktopStoryTests, } from './storyTests.js'; +export { desktopRequirements, matchesDesktopPlatforms, resolveDesktopInput, validateDesktopCaseOptions } from './input.js'; +export type { + DesktopCaseOptions, + DesktopInput, + DesktopInputLane, + DesktopPlatforms, + DesktopRequirement, + ResolvedDesktopCaseOptions, +} from './input.js'; export type { DesktopStoryCapability, DesktopStoryExpectation, diff --git a/packages/agentic/desktop-driver/src/authoring/input.test.ts b/packages/agentic/desktop-driver/src/authoring/input.test.ts new file mode 100644 index 0000000000..1c02abba19 --- /dev/null +++ b/packages/agentic/desktop-driver/src/authoring/input.test.ts @@ -0,0 +1,32 @@ +import { matchesDesktopPlatforms, resolveDesktopInput, validateDesktopCaseOptions } from './input.js'; + +describe('desktop input descriptors', () => { + test.each([ + [undefined, [], 'both'], + [undefined, ['click'], 'both'], + [undefined, ['physical-click'], 'physical'], + [undefined, ['accessibility-click'], 'accessible'], + ['accessible', [], 'accessible'], + ['physical', ['click'], 'physical'], + ] as const)('derives input from %s and %s', (input, requires, expected) => { + expect(resolveDesktopInput(input, requires)).toBe(expected); + }); + + test('rejects mixed transports and explicit widening', () => { + expect(() => resolveDesktopInput(undefined, ['keyboard', 'accessibility-click'])).toThrow('split the case'); + expect(() => resolveDesktopInput('both', ['physical-click'])).toThrow('cannot widen'); + expect(() => resolveDesktopInput('physical', ['accessibility-click'])).toThrow('cannot widen'); + }); + + test('validates platform filters and negation', () => { + expect(matchesDesktopPlatforms(['!win32'], 'windows')).toBe(true); + expect(matchesDesktopPlatforms(['!win32'], 'win32')).toBe(false); + expect(() => validateDesktopCaseOptions({ platforms: ['macos', '!win32'] })).toThrow('known'); + expect(() => validateDesktopCaseOptions({ platforms: ['!macos', '!windows', '!win32'] })).toThrow('every platform'); + }); + + test('rejects duplicate and unknown requirements', () => { + expect(() => validateDesktopCaseOptions({ requires: ['click', 'click'] })).toThrow('duplicate'); + expect(() => validateDesktopCaseOptions({ requires: ['input'] })).toThrow('unsupported requirement'); + }); +}); diff --git a/packages/agentic/desktop-driver/src/authoring/input.ts b/packages/agentic/desktop-driver/src/authoring/input.ts new file mode 100644 index 0000000000..ee0e353c51 --- /dev/null +++ b/packages/agentic/desktop-driver/src/authoring/input.ts @@ -0,0 +1,167 @@ +import type { DesktopEndpoint } from '../protocol/types.js'; + +export const desktopRequirements = [ + 'accessibility-actions', + 'accessibility-click', + 'click', + 'element-screenshot', + 'focus', + 'keyboard', + 'physical-click', + 'screenshot', + 'set-window-rect', + 'wheel', +] as const; + +export type DesktopRequirement = (typeof desktopRequirements)[number]; +export type DesktopInput = 'accessible' | 'physical' | 'both'; +export type DesktopInputLane = Exclude; +export type DesktopPlatforms = + | readonly [DesktopEndpoint, ...DesktopEndpoint[]] + | readonly [`!${DesktopEndpoint}`, ...`!${DesktopEndpoint}`[]]; + +export type DesktopCaseOptions = { + input?: DesktopInput; + requires?: readonly DesktopRequirement[]; + platforms?: DesktopPlatforms; + scope?: Readonly>; + unsupported?: { platforms: DesktopPlatforms; reason: string; issue?: string }; + quarantine?: { + issue: string; + reason: string; + expires: string; + platforms?: DesktopPlatforms; + input?: DesktopInputLane; + }; +}; + +export type ResolvedDesktopCaseOptions = DesktopCaseOptions & { + input: DesktopInput; + inputSource: 'declared' | 'derived'; +}; + +const physicalRequirements = new Set(['physical-click', 'keyboard', 'wheel']); +const accessibleRequirements = new Set(['accessibility-click']); +const endpoints = ['macos', 'windows', 'win32'] as const; +const requirementSet = new Set(desktopRequirements); + +export function resolveDesktopInput(input?: DesktopInput, requires: readonly DesktopRequirement[] = []): DesktopInput { + const physical = requires.some((requirement) => physicalRequirements.has(requirement)); + const accessible = requires.some((requirement) => accessibleRequirements.has(requirement)); + if (physical && accessible) { + throw new TypeError('A desktop test cannot require both physical and accessible input; split the case.'); + } + const derived = physical ? 'physical' : accessible ? 'accessible' : 'both'; + if (input !== undefined && derived !== 'both' && input !== derived) { + throw new TypeError(`input "${input}" cannot widen the modality derived from requires ("${derived}").`); + } + return input ?? derived; +} + +export function validateDesktopCaseOptions(value: unknown, source = 'desktop test options'): ResolvedDesktopCaseOptions { + const options = object(value, source); + exactKeys(options, ['input', 'requires', 'platforms', 'scope', 'unsupported', 'quarantine'], source); + if (options.input !== undefined && !['accessible', 'physical', 'both'].includes(options.input as string)) { + throw new TypeError(`${source}.input must be accessible, physical, or both.`); + } + const requires: DesktopRequirement[] = []; + if (options.requires !== undefined) { + if (!Array.isArray(options.requires)) throw new TypeError(`${source}.requires must be an array.`); + const seen = new Set(); + for (const requirement of options.requires) { + if (typeof requirement !== 'string' || !requirementSet.has(requirement)) { + throw new TypeError(`${source}.requires contains an unsupported requirement.`); + } + if (seen.has(requirement)) throw new TypeError(`${source}.requires contains duplicate "${requirement}".`); + seen.add(requirement); + requires.push(requirement as DesktopRequirement); + } + } + validatePlatforms(options.platforms, `${source}.platforms`); + validateScope(options.scope, `${source}.scope`); + if (options.unsupported !== undefined) { + const unsupported = object(options.unsupported, `${source}.unsupported`); + exactKeys(unsupported, ['platforms', 'reason', 'issue'], `${source}.unsupported`); + validatePlatforms(unsupported.platforms, `${source}.unsupported.platforms`, true); + nonEmptyString(unsupported.reason, `${source}.unsupported.reason`); + if (unsupported.issue !== undefined) nonEmptyString(unsupported.issue, `${source}.unsupported.issue`); + } + if (options.quarantine !== undefined) { + const quarantine = object(options.quarantine, `${source}.quarantine`); + exactKeys(quarantine, ['issue', 'reason', 'expires', 'platforms', 'input'], `${source}.quarantine`); + nonEmptyString(quarantine.issue, `${source}.quarantine.issue`); + nonEmptyString(quarantine.reason, `${source}.quarantine.reason`); + if (typeof quarantine.expires !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(quarantine.expires)) { + throw new TypeError(`${source}.quarantine.expires must be an ISO date.`); + } + if (quarantine.input !== undefined && !['accessible', 'physical'].includes(quarantine.input as string)) { + throw new TypeError(`${source}.quarantine.input must be accessible or physical.`); + } + validatePlatforms(quarantine.platforms, `${source}.quarantine.platforms`); + } + const resolvedInput = resolveDesktopInput(options.input as DesktopInput | undefined, requires); + return { + ...(options.input === undefined ? {} : { input: options.input as DesktopInput }), + ...(options.requires === undefined ? {} : { requires }), + ...(options.platforms === undefined ? {} : { platforms: options.platforms as DesktopPlatforms }), + ...(options.scope === undefined ? {} : { scope: options.scope as DesktopCaseOptions['scope'] }), + ...(options.unsupported === undefined ? {} : { unsupported: options.unsupported as DesktopCaseOptions['unsupported'] }), + ...(options.quarantine === undefined ? {} : { quarantine: options.quarantine as DesktopCaseOptions['quarantine'] }), + input: resolvedInput, + inputSource: options.input === undefined ? 'derived' : 'declared', + }; +} + +export function matchesDesktopPlatforms(platforms: DesktopPlatforms | undefined, endpoint: DesktopEndpoint): boolean { + if (platforms === undefined) return true; + const negative = platforms[0].startsWith('!'); + const values = platforms.map((platform) => (negative ? platform.slice(1) : platform)); + return negative ? !values.includes(endpoint) : values.includes(endpoint); +} + +function validatePlatforms(value: unknown, source: string, required = false): void { + if (value === undefined && !required) return; + if (!Array.isArray(value) || value.length === 0) throw new TypeError(`${source} must be a non-empty platform list.`); + const negative = typeof value[0] === 'string' && value[0].startsWith('!'); + const seen = new Set(); + for (const entry of value) { + if (typeof entry !== 'string') throw new TypeError(`${source} contains an unsupported platform.`); + const platform = negative ? entry.slice(1) : entry; + if (entry.startsWith('!') !== negative || !endpoints.includes(platform as (typeof endpoints)[number])) { + throw new TypeError(`${source} must contain only known ${negative ? 'negated ' : ''}platforms.`); + } + if (seen.has(platform)) throw new TypeError(`${source} contains duplicate "${entry}".`); + seen.add(platform); + } + if (negative && seen.size === endpoints.length) throw new TypeError(`${source} cannot exclude every platform.`); +} + +function validateScope(value: unknown, source: string): void { + if (value === undefined) return; + const scope = object(value, source); + for (const [key, values] of Object.entries(scope)) { + if (!/^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)*$/.test(key)) throw new TypeError(`${source} contains invalid key "${key}".`); + if (!Array.isArray(values) || values.length === 0) throw new TypeError(`${source}.${key} must be a non-empty value list.`); + const seen = new Set(); + for (const item of values) { + if (typeof item !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(item)) { + throw new TypeError(`${source}.${key} contains an invalid value.`); + } + if (seen.has(item)) throw new TypeError(`${source}.${key} contains duplicate "${item}".`); + seen.add(item); + } + } +} + +function object(value: unknown, source: string): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) throw new TypeError(`${source} must be an object.`); + return value as Record; +} + +function exactKeys(value: Record, allowed: readonly string[], source: string): void { + for (const key of Object.keys(value)) if (!allowed.includes(key)) throw new TypeError(`${source} contains unknown field "${key}".`); +} + +function nonEmptyString(value: unknown, source: string): void { + if (typeof value !== 'string' || !value.trim()) throw new TypeError(`${source} must be a non-empty string.`); +} diff --git a/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.test.ts b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.test.ts index ddcf1c77c9..710571be26 100644 --- a/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.test.ts +++ b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.test.ts @@ -8,6 +8,7 @@ import { FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES, NativeDesktopHost, parseKeyedActions, + resolveDisabledInputFeatures, translateNativeError, } from './NativeDesktopHost'; @@ -20,6 +21,19 @@ describe('NativeDesktopHost accessibility action keys', () => { expect(parseKeyedActions({ actions: [], keys: [] })).toEqual([]); }); + describe('NativeDesktopHost input feature policy', () => { + test('unions explicit masks with the operator environment mask', () => { + const previous = process.env[FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES]; + process.env[FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES] = 'physicalClick,wheel'; + try { + expect([...resolveDisabledInputFeatures(['keyboard'])].sort()).toEqual(['keyboard', 'physicalClick', 'wheel']); + } finally { + if (previous === undefined) delete process.env[FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES]; + else process.env[FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES] = previous; + } + }); + }); + test.each([ ['no keys', { actions: ['AXPress'] }], ['lists of different lengths', { actions: ['AXPress', 'Toggle'], keys: ['k1'] }], diff --git a/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.ts b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.ts index 8bd1da92d2..4c33f0aeb4 100644 --- a/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.ts +++ b/packages/agentic/desktop-driver/src/hosts/native/NativeDesktopHost.ts @@ -497,14 +497,14 @@ export function applyDisabledInputFeaturePolicy( return { ...hostInfo, features }; } -function resolveDisabledInputFeatures(configured?: readonly NativeDisabledInputFeature[]): ReadonlySet { - const values = - configured ?? - process.env[FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES] +export function resolveDisabledInputFeatures(configured?: readonly NativeDisabledInputFeature[]): ReadonlySet { + const values = [ + ...(configured ?? []), + ...(process.env[FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES] ?.split(',') .map((value) => value.trim()) - .filter(Boolean) ?? - []; + .filter(Boolean) ?? []), + ]; const allowed = new Set(['keyboard', 'physicalClick', 'wheel']); const resolved = new Set(); for (const value of values) { diff --git a/packages/agentic/desktop-driver/src/index.ts b/packages/agentic/desktop-driver/src/index.ts index 5c66f111c9..3e556374bc 100644 --- a/packages/agentic/desktop-driver/src/index.ts +++ b/packages/agentic/desktop-driver/src/index.ts @@ -32,6 +32,15 @@ export { desktopStoryPlatforms, validateDesktopStoryTests, } from './authoring/storyTests.js'; +export { desktopRequirements, matchesDesktopPlatforms, resolveDesktopInput, validateDesktopCaseOptions } from './authoring/input.js'; +export type { + DesktopCaseOptions, + DesktopInput, + DesktopInputLane, + DesktopPlatforms, + DesktopRequirement, + ResolvedDesktopCaseOptions, +} from './authoring/input.js'; export type { DesktopStoryCapability, DesktopStoryExpectation, diff --git a/packages/agentic/desktop-driver/src/storybook.ts b/packages/agentic/desktop-driver/src/storybook.ts index 4946ad486e..9aa50d9d49 100644 --- a/packages/agentic/desktop-driver/src/storybook.ts +++ b/packages/agentic/desktop-driver/src/storybook.ts @@ -1,4 +1,5 @@ import type { DesktopStoryTests } from './authoring/storyTests.js'; +import type { ResolvedDesktopCaseOptions } from './authoring/input.js'; import type { DesktopEndpoint } from './protocol/types.js'; export type DesktopStoryManifestEntry = { @@ -9,7 +10,12 @@ export type DesktopStoryManifestEntry = { tags: readonly string[]; title: string; tests?: DesktopStoryTests; - wdio?: { digest: string; exportName: string; testNames?: readonly string[] }; + wdio?: { + digest: string; + exportName: string; + testNames?: readonly string[]; + cases?: readonly { name: string; options: ResolvedDesktopCaseOptions }[]; + }; }; export type DesktopStoryManifest = { diff --git a/packages/agentic/storybook-desktop/config/transform-story-tests.cjs b/packages/agentic/storybook-desktop/config/transform-story-tests.cjs index aac57cf91a..bc47137db0 100644 --- a/packages/agentic/storybook-desktop/config/transform-story-tests.cjs +++ b/packages/agentic/storybook-desktop/config/transform-story-tests.cjs @@ -57,6 +57,55 @@ function resolveInitializer(value, seen = new Set(), bindings = new Set()) { return value; } +function importedDesktopIt(call) { + const callee = call.get('callee'); + if (!callee.isIdentifier()) fail(call, 'array tests must be direct desktopIt imports.'); + const binding = callee.scope.getBinding(callee.node.name); + const specifier = binding?.path; + const declaration = specifier?.parentPath; + if ( + !specifier?.isImportSpecifier() || + (specifier.node.imported.name ?? specifier.node.imported.value) !== 'desktopIt' || + !declaration?.isImportDeclaration() || + declaration.node.source.value !== '@fluentui-react-native/storybook-desktop/testing' || + declaration.node.importKind === 'type' || + specifier.node.importKind === 'type' + ) { + fail(call, 'array tests must use the desktopIt value imported from @fluentui-react-native/storybook-desktop/testing.'); + } +} + +function staticValue(value) { + value = unwrap(value); + if (value.isStringLiteral() || value.isBooleanLiteral() || value.isNullLiteral()) return value.node.value; + if (value.isNumericLiteral() && Number.isFinite(value.node.value)) return value.node.value; + if (value.isTemplateLiteral() && value.node.expressions.length === 0 && value.node.quasis.length === 1) { + return value.node.quasis[0].value.cooked; + } + if (value.isUnaryExpression({ operator: '-' }) && value.get('argument').isNumericLiteral()) { + return -value.node.argument.value; + } + if (value.isArrayExpression()) { + return value.get('elements').map((element) => { + if (!element?.node || element.isSpreadElement()) fail(element ?? value, 'descriptor arrays must contain static values.'); + return staticValue(element); + }); + } + if (value.isObjectExpression()) { + const result = {}; + for (const property of value.get('properties')) { + if (!property.isObjectProperty() || property.node.computed || property.node.shorthand) { + fail(property, 'descriptor objects require static key/value properties.'); + } + const key = keyName(property); + if (!key || Object.hasOwn(result, key)) fail(property, 'descriptor objects require unique static property names.'); + result[key] = staticValue(property.get('value')); + } + return result; + } + fail(value, 'descriptor options must be a strict JSON literal.'); +} + function collectCallbacks(value) { const requireCallback = (callback) => { if ((!callback.isArrowFunctionExpression() && !callback.isFunctionExpression()) || callback.node.generator) { @@ -64,6 +113,29 @@ function collectCallbacks(value) { } return callback; }; + if (value.isArrayExpression()) { + const callbacks = []; + const names = new Set(); + for (const item of value.get('elements')) { + if (!item?.node || item.isSpreadElement() || !item.isCallExpression()) { + fail(item ?? value, 'desktopIt arrays require direct calls and cannot contain holes or spreads.'); + } + importedDesktopIt(item); + const args = item.get('arguments'); + if (args.length < 2 || args.length > 3 || args.some((argument) => argument.isSpreadElement())) { + fail(item, 'desktopIt requires a name, optional static options, and an inline callback.'); + } + const name = args[0].isStringLiteral() ? args[0].node.value : undefined; + if (!name?.trim()) fail(args[0], 'desktopIt requires a non-empty literal test name.'); + if (names.has(name)) fail(args[0], `duplicate test name "${name}".`); + names.add(name); + const options = args.length === 2 ? {} : staticValue(args[1]); + if (!options || typeof options !== 'object' || Array.isArray(options)) fail(args[1], 'desktopIt options must be an object.'); + callbacks.push({ name, options, callback: requireCallback(unwrap(args[args.length - 1])) }); + } + if (!callbacks.length) fail(value, 'a desktopIt array must contain at least one test.'); + return callbacks; + } if (!value.isObjectExpression()) return [{ callback: requireCallback(value) }]; const callbacks = []; const names = new Set(); @@ -276,6 +348,17 @@ module.exports = () => ({ for (const { callback } of callbacks) compileWdioTest(callback, filename, state.file.code); } for (const { property } of tests.values()) if (!property.removed) property.remove(); + program.scope.crawl(); + for (const statement of program.get('body')) { + if (!statement.isImportDeclaration() || statement.node.source.value !== '@fluentui-react-native/storybook-desktop/testing') + continue; + for (const specifier of statement.get('specifiers')) { + if (!specifier.isImportSpecifier() || (specifier.node.imported.name ?? specifier.node.imported.value) !== 'desktopIt') continue; + const binding = statement.scope.getBinding(specifier.node.local.name); + if (!binding?.referencePaths.length) specifier.remove(); + } + if (!statement.node.specifiers.length) statement.remove(); + } }, }, }); diff --git a/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts b/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts index 8444ed0b64..67e6ed935c 100644 --- a/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts +++ b/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts @@ -5,12 +5,13 @@ declare function storyTests(): PluginObj; declare namespace storyTests { type Callback = { name?: string; + options?: Record; callback: NodePath; }; type StoryTest = { callbacks: readonly Callback[]; property: NodePath; - value: NodePath; + value: NodePath; }; function collectWdioTests(program: NodePath): ReadonlyMap; diff --git a/packages/agentic/storybook-desktop/references/commands.md b/packages/agentic/storybook-desktop/references/commands.md index 27e2975392..f60208c21c 100644 --- a/packages/agentic/storybook-desktop/references/commands.md +++ b/packages/agentic/storybook-desktop/references/commands.md @@ -28,6 +28,7 @@ only). The mode precedence is CLI flag, `DESKTOP_REPORTER`, config, then | `instance --` | Inspect the active instance's ports and target | | `test -- --list` | Discover authored executable story-test IDs | | `test -- --test ` | Run matching isolated story callbacks | +| `test -- --input accessible` | Select accessible and `both` descriptor cases | | `smoke -- --mode stories` | Build/bundle/launch, traverse stories, clean up | | `smoke -- --mode stories-and-tests` | Traverse stories and execute callbacks | @@ -60,6 +61,13 @@ Metro, generated runtime identity, and exact process cleanup. Windows RN Test App retains its required Metro port 8081; do not run concurrent Metro-backed Windows smokes. +Storybook test and stories-and-tests smoke commands accept +`--input accessible|physical`. The lane filters `desktopIt` cases by their +resolved input and target platform. `driver --input` records the default lane +for subsequent tests against that instance. Legacy callback collections are +not eligible for input-lane runs. Input lanes cannot be combined with an +explicit non-`auto` click mode. + The TypeScript API exposes the same operations through `DesktopStorybookCli`. Higher-level supervisors should inject their shared reporter, runner, and services so runs join one owned lifecycle. diff --git a/packages/agentic/storybook-desktop/references/story-tests.md b/packages/agentic/storybook-desktop/references/story-tests.md index 1fd21d7e2e..62662edbbd 100644 --- a/packages/agentic/storybook-desktop/references/story-tests.md +++ b/packages/agentic/storybook-desktop/references/story-tests.md @@ -7,11 +7,12 @@ owner_paths: # Author story tests -Story tests are statically extracted CSF `wdio` callbacks. Use top-level -functions or named callback collections with literal names; callbacks must not -close over module-local values. Import Node-only helpers through literal -dynamic `import()` inside the callback. Keep test code out of the native -bundle. +Story tests are statically extracted CSF `wdio` callbacks. Use `desktopIt` +descriptors when declaring input requirements or endpoint scope. Top-level +functions or named callback collections remain supported for compatibility, +but cannot be selected by an input lane. Callbacks must not close over +module-local values. Import Node-only helpers through literal dynamic +`import()` inside the callback. Keep test code out of the native bundle. ```tsx import type { StoryObj } from '@storybook/react-native'; @@ -37,6 +38,33 @@ export const Default: Story = { }; ``` +For modality-aware tests, import the value helper and use an array of cases: + +```tsx +import { desktopIt, type WdioStory } from '@fluentui-react-native/storybook-desktop/testing'; + +export const Default: WdioStory> = { + args: { testID: 'save-button' }, + wdio: [ + desktopIt('activates in either input lane', { requires: ['click'] }, async ({ browser, expect }) => { + await (await browser.$('~save-button')).click(); + await expect(await browser.$('~saved-message')).toBeDisplayed(); + }), + desktopIt('uses a physical pointer', { requires: ['physical-click'], platforms: ['win32'] }, async ({ browser }) => { + await (await browser.$('~save-button')).click(); + }), + ], +}; +``` + +`requires: ['click']` derives to `both` and adapts `element.click()` to the +selected lane: accessibility click in the accessible lane and physical click +in the physical lane. Explicit physical or accessibility requirements derive +their matching lane. `input` may narrow a `both` case but cannot widen a +transport-specific requirement. `platforms` is an allowlist by default; +`['!win32']` excludes that endpoint. Platform selection uses the target +endpoint, so Win32 remains distinct from Windows. + The callback receives the real WebdriverIO `browser`, `expect-webdriverio` `expect`, typed `desktop` assertions/story commands, target `platform` (`macos`, `windows`, or `win32`), cooperative `signal`, and explicit `skip(reason)`. @@ -73,6 +101,13 @@ editing callbacks because their digest participates in manifest freshness. filters names. `test` operates against an already running app/driver; smoke owns the complete app lifecycle. +Use `storybook test --input accessible|physical` or +`storybook smoke --mode stories-and-tests --input accessible|physical` to +select declared cases. `storybook driver --input ...` supplies the default +lane to a later `storybook test` invocation. `--input` cannot be combined +with a non-default `--click-mode`. A lane requires descriptor cases and rejects +legacy callback collections rather than silently treating them as `both`. + Use the component authoring contract for component-specific test assertions. Legacy `parameters.desktopDriver` static plans remain for compatibility and are not the preferred story-test authoring API. diff --git a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts index 5a199c7cda..ad62fc01bf 100644 --- a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts +++ b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts @@ -48,6 +48,7 @@ import type { DesktopCommandRunner, PreparedDesktopCommand, RunningDesktopComman import { formatDesktopStorybookSmokeTestSummary, runDesktopStorybookSmokeTests } from './smokeTests.js'; import { matchesWdioTest, runWdioStoryTests, selectWdioStories } from '../testing/runWdioTests.js'; import { resolveWdioOptions, type DesktopStorybookWdioOptions } from '../config/wdio.js'; +import { matchesDesktopPlatforms } from '@fluentui-react-native/desktop-driver/authoring'; type ResolvedDesktopStorybookInstance = DesktopStorybookInstance & { driverManifestPath?: string; @@ -92,6 +93,7 @@ export type DesktopStorybookServerOptions = { }; export type DesktopStorybookDriverOptions = DesktopStorybookServerOptions & { + input?: 'accessible' | 'physical'; /** * Protects the driver with a grant authority and publishes one agent's connection descriptor, which the Desktop * Driver MCP adapter reads. WDIO `storybook test` cannot connect to an agent-served driver. @@ -219,6 +221,7 @@ export class DesktopStorybookCli { async driver(platform: Platforms, options: DesktopStorybookDriverOptions = {}): Promise { validateServerOptions(options); validateDriverHost(options.host); + const wdio = resolveWdioOptions({ ...this.config.wdio, ...(options.input ? { input: options.input } : {}) }); const agent = options.agent && { descriptor: options.agent.descriptor ?? defaultAgentDescriptor, // The channel service runs in another working directory. @@ -244,6 +247,7 @@ export class DesktopStorybookCli { resolvedInstance, nativeDriver, agent && { ...agent, ...(agentApplication ? { application: agentApplication } : {}) }, + wdio, ); resolvedInstance.driverManifestPath = driverManifest.path; this.services.start('metro', this.prepareCommand(defaultMetroCommand(resolvedInstance), platform, resolvedInstance)); @@ -329,16 +333,26 @@ export class DesktopStorybookCli { } async test(platform: Platforms, options: DesktopStorybookTestOptions = {}): Promise { - const settings = resolveWdioOptions({ + const overrides: DesktopStorybookWdioOptions = { + ...(options.clickMode === undefined ? {} : { clickMode: options.clickMode }), + ...(options.input === undefined ? {} : { input: options.input }), + ...(options.testReporter === undefined ? {} : { reporter: options.testReporter }), + ...(options.story === undefined ? {} : { story: options.story }), + ...(options.tag === undefined ? {} : { tag: options.tag }), + ...(options.test === undefined ? {} : { test: options.test }), + ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }), + }; + const initialSettings = resolveWdioOptions({ ...this.config.wdio, - ...Object.fromEntries( - Object.entries({ ...options, reporter: options.testReporter }).filter( - ([key, value]) => key !== 'testReporter' && value !== undefined, - ), - ), + ...overrides, }); + const connection = options.list ? undefined : this.testConnection(platform, options); + const settings = + connection?.input && initialSettings.input === undefined + ? resolveWdioOptions({ ...initialSettings, input: connection.input }) + : initialSettings; const manifest = await this.createStoryManifest(this.config, platform); - const selected = selectWdioStories(manifest, settings); + const selected = selectWdioStories(manifest, settings, platform); if (selected.length === 0) { throw new Error('No executable wdio story tests matched the requested selection.'); } @@ -346,13 +360,32 @@ export class DesktopStorybookCli { const value = selected.map(({ id, sourcePath, wdio }) => ({ id, sourcePath, - ...(wdio?.testNames ? { tests: wdio.testNames.filter((name) => matchesWdioTest(name, settings.test)) } : {}), + ...(wdio?.cases + ? { + tests: wdio.cases + .filter( + ({ name, options: descriptor }) => + matchesWdioTest(name, settings.test) && + (!settings.input || descriptor.input === 'both' || descriptor.input === settings.input) && + matchesDesktopPlatforms(descriptor.platforms, platform), + ) + .map(({ name, options: descriptor }) => ({ + name, + input: descriptor.input, + inputSource: descriptor.inputSource, + requires: descriptor.requires ?? [], + platforms: descriptor.platforms, + })), + } + : wdio?.testNames + ? { tests: wdio.testNames.filter((name) => matchesWdioTest(name, settings.test)) } + : {}), })); if (this.reporterMode === 'json' || this.reporterMode === 'ndjson') this.writeStructured(value); else this.output.write(`${JSON.stringify(value, null, 2)}\n`); return; } - const connection = this.testConnection(platform, options); + const activeConnection = connection ?? this.testConnection(platform, options); if (platform === 'macos' && !options.url) { await this.writeMacOSApplicationLease( path.join(this.config.projectRoot, 'storybook-desktop.generated', 'driver-manifest.macos.json'), @@ -362,13 +395,14 @@ export class DesktopStorybookCli { const results = await runWdioStoryTests( { ...settings, - ...connection, + ...activeConnection, config: this.config, manifest, platform, errorOutput: this.errorOutput, report: this.reporter, signal: this.services.controller.signal, + input: settings.input, }, this.runner, ); @@ -378,7 +412,10 @@ export class DesktopStorybookCli { ); } - private testConnection(platform: Platforms, options: DesktopStorybookTestOptions): { targetId: string; url: string } { + private testConnection( + platform: Platforms, + options: DesktopStorybookTestOptions, + ): { input?: 'accessible' | 'physical'; targetId: string; url: string } { if (options.url || options.target) { if (!options.url || !options.target) { throw new Error('Pass --url and --target together, or omit both to use the running Storybook driver instance.'); @@ -415,7 +452,17 @@ export class DesktopStorybookCli { `The Storybook driver instance at ${manifestPath} serves an agent; restart storybook driver --${platform} without --agent to run storybook test.`, ); } - return { targetId: manifest.targetId, url: loopbackUrl(manifest.driverPort) }; + const input = + typeof manifest === 'object' && + manifest !== null && + 'wdio' in manifest && + typeof manifest.wdio === 'object' && + manifest.wdio !== null && + 'input' in manifest.wdio && + (manifest.wdio.input === 'accessible' || manifest.wdio.input === 'physical') + ? manifest.wdio.input + : undefined; + return { targetId: manifest.targetId, url: loopbackUrl(manifest.driverPort), ...(input ? { input } : {}) }; } printInstance(platform: Platforms): void { @@ -478,8 +525,15 @@ export class DesktopStorybookCli { throw unsupportedAction('smoke', platform); } if (smoke?.command) { - const instance = await this.resolveSmokeInstance(platform, smoke, mode); - await this.executePlan(withEnvironment(smoke.command, { [STORYBOOK_SMOKE_MODE]: mode }), platform, instance); + const instance = await this.resolveSmokeInstance(platform, smoke, mode, options.input); + await this.executePlan( + withEnvironment(smoke.command, { + [STORYBOOK_SMOKE_MODE]: mode, + ...(options.input ? { DESKTOP_INPUT: options.input } : {}), + }), + platform, + instance, + ); return; } if (!smoke?.stop) { @@ -488,8 +542,8 @@ export class DesktopStorybookCli { ); } - const instance = await this.resolveSmokeInstance(platform, smoke, mode); - await this.runSmokeLifecycle(platform, smoke, instance, mode); + const instance = await this.resolveSmokeInstance(platform, smoke, mode, options.input); + await this.runSmokeLifecycle(platform, smoke, instance, mode, options.input); } private async executeAction( @@ -509,6 +563,7 @@ export class DesktopStorybookCli { smoke: DesktopSmokeOptions, instance: ResolvedDesktopStorybookInstance, mode: DesktopSmokeMode, + input?: 'accessible' | 'physical', ): Promise { const failures: unknown[] = []; let primaryFailure: unknown; @@ -571,6 +626,7 @@ export class DesktopStorybookCli { targetId: this.targetId(platform), report: this.reporter, signal: this.services.controller.signal, + input, }), ); this.output.write(`${formatDesktopStorybookSmokeTestSummary(result)}\n`); @@ -624,6 +680,7 @@ export class DesktopStorybookCli { platform: Platforms, smoke: DesktopSmokeOptions, mode: DesktopSmokeMode, + input?: 'accessible' | 'physical', ): Promise { const storybookPort = smoke.serverUrl ? portFromUrl(smoke.serverUrl) @@ -641,7 +698,10 @@ export class DesktopStorybookCli { if (mode === 'stories-and-tests') { const nativeDriver = await this.resolveDriver(platform); - resolvedInstance.driverManifestPath = (await this.writeDriverManifest(platform, resolvedInstance, nativeDriver)).path; + const wdio = resolveWdioOptions({ ...this.config.wdio, ...(input ? { input } : {}) }); + resolvedInstance.driverManifestPath = ( + await this.writeDriverManifest(platform, resolvedInstance, nativeDriver, undefined, wdio) + ).path; } if (platform === 'macos') { resolvedInstance.macosXcconfigPath = writeMacOSInstanceConfig(this.config.projectRoot, resolvedInstance); @@ -659,6 +719,7 @@ export class DesktopStorybookCli { instance: DesktopStorybookInstance, nativeDriver: NativeDriverArtifact, agent?: CreateDesktopStorybookDriverManifestOptions['agent'], + wdio?: CreateDesktopStorybookDriverManifestOptions['wdio'], ): Promise<{ manifest: DesktopStorybookDriverManifest; path: string }> { const storyManifest = await this.createStoryManifest(this.config, platform); const outputPath = path.join(this.config.projectRoot, 'storybook-desktop.generated', `driver-manifest.${platform}.json`); @@ -672,6 +733,7 @@ export class DesktopStorybookCli { platform, renderer, storyManifest, + ...(wdio ? { wdio } : {}), }); writeDesktopStorybookDriverManifest(driverManifest, outputPath); return { manifest: driverManifest, path: outputPath }; diff --git a/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts b/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts index 263ab8e307..f7552ff94f 100644 --- a/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts +++ b/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts @@ -36,9 +36,9 @@ type PlatformFlags = { }; type ServerFlags = PlatformFlags & DesktopStorybookServerOptions; -type DriverFlags = ServerFlags & { agent?: boolean; descriptor?: string; root?: string }; +type DriverFlags = ServerFlags & { agent?: boolean; descriptor?: string; root?: string; input?: 'accessible' | 'physical' }; type ManifestFlags = PlatformFlags & { out?: string }; -type SmokeFlags = PlatformFlags & { mode: DesktopSmokeMode }; +type SmokeFlags = PlatformFlags & { mode: DesktopSmokeMode; input?: 'accessible' | 'physical' }; type BuildDriverFlags = PlatformFlags & DesktopStorybookBuildDriverOptions; type PrepFlags = PlatformFlags & { driver: DesktopStorybookPrepOptions['driver'] }; type GetApi = (flags: PlatformFlags) => Promise; @@ -127,14 +127,19 @@ function addTestCommand(program: Command, getApi: GetApi): void { .option('--tag ', 'filter story tags') .option('--url ', 'Desktop Driver URL (requires --target)') .option('--target ', 'registered target ID (requires --url)') + .addOption(new Option('--input ', 'select accessible or physical input tests').choices(['accessible', 'physical'])) .option('--timeout-ms ', 'timeout per executable test', Number) .addOption(new Option('--test-reporter ', 'Node test reporter').choices(['spec', 'tap', 'dot'])) .addOption(new Option('--click-mode ', 'native click semantics').choices(['auto', 'physical', 'accessibility'])); addPlatformOptions(command); command.action(async (flags: PlatformFlags & DesktopStorybookTestOptions) => { + if (flags.input !== undefined && flags.clickMode !== undefined) { + throw new TypeError('--input cannot be combined with --click-mode.'); + } const api = await getApi(flags); await api.test(resolvePlatform(flags, api), { clickMode: flags.clickMode, + input: flags.input, list: flags.list, testReporter: flags.testReporter, story: flags.story, @@ -178,6 +183,7 @@ function addDriverCommand(program: Command, getApi: GetApi): void { .option('--agent', 'require a grant on every driver route and publish one agent connection descriptor') .option('--descriptor ', 'agent connection descriptor record name (default agent-connection.json)') .option('--root ', 'private storage root for the agent descriptor; defaults to the platform location'); + command.addOption(new Option('--input ', 'default input lane for Storybook tests').choices(['accessible', 'physical'])); addPlatformOptions(command); command.action(async (flags: DriverFlags) => { if (!flags.agent && (flags.descriptor !== undefined || flags.root !== undefined)) { @@ -187,6 +193,7 @@ function addDriverCommand(program: Command, getApi: GetApi): void { await api.driver(resolvePlatform(flags, api), { host: flags.host, port: flags.port, + input: flags.input, ...(flags.agent ? { agent: { @@ -387,10 +394,14 @@ function addSmokeCommand(program: Command, getApi: GetApi): void { .choices([...desktopSmokeModes]) .default('stories'), ); + command.addOption(new Option('--input ', 'select accessible or physical input tests').choices(['accessible', 'physical'])); addPlatformOptions(command); command.action(async (flags: SmokeFlags) => { + if (flags.input !== undefined && flags.mode !== 'stories-and-tests') { + throw new TypeError('--input requires --mode stories-and-tests.'); + } const api = await getApi(flags); - await api.smoke(resolvePlatform(flags, api), { mode: flags.mode }); + await api.smoke(resolvePlatform(flags, api), { mode: flags.mode, input: flags.input }); }); } diff --git a/packages/agentic/storybook-desktop/src/cli/smokeTests.ts b/packages/agentic/storybook-desktop/src/cli/smokeTests.ts index 2dcfb9a1f2..afa53f720f 100644 --- a/packages/agentic/storybook-desktop/src/cli/smokeTests.ts +++ b/packages/agentic/storybook-desktop/src/cli/smokeTests.ts @@ -21,6 +21,7 @@ type DesktopStorybookSmokeSession = Pick Promise; export type DesktopStorybookSmokeTestOptions = { + input?: 'accessible' | 'physical'; artifactsRoot?: string; config: DesktopStorybookConfig; driverUrl: string; @@ -52,13 +53,14 @@ export async function runDesktopStorybookSmokeTests( const testTag = options.testTag ?? process.env.STORYBOOK_SMOKE_TAG; const wdioSettings = resolveWdioOptions({ ...config.wdio, + ...(options.input ? { input: options.input } : {}), ...(storyPattern !== undefined ? { story: storyPattern } : {}), ...(testTag !== undefined ? { tag: testTag } : {}), }); const planStoryIds = new Set( selectDesktopStoryTests(manifest, platform, { story: storyPattern, tag: testTag ?? 'desktop-e2e' }).map(({ entry }) => entry.id), ); - const wdioStoryIds = new Set(selectWdioStories(manifest, wdioSettings).map(({ id }) => id)); + const wdioStoryIds = new Set(selectWdioStories(manifest, wdioSettings, platform).map(({ id }) => id)); const storyIds = [...new Set([...planStoryIds, ...wdioStoryIds])].sort(); if (storyIds.length === 0) { throw new Error('No desktop story tests matched the smoke selection.'); diff --git a/packages/agentic/storybook-desktop/src/config/commands.ts b/packages/agentic/storybook-desktop/src/config/commands.ts index 1061ef9350..7df2609e57 100644 --- a/packages/agentic/storybook-desktop/src/config/commands.ts +++ b/packages/agentic/storybook-desktop/src/config/commands.ts @@ -24,6 +24,7 @@ export type DesktopSmokeRunOptions = { * @default "stories" */ mode?: DesktopSmokeMode; + input?: 'accessible' | 'physical'; }; export const STORYBOOK_SMOKE_MODE = 'STORYBOOK_SMOKE_MODE'; diff --git a/packages/agentic/storybook-desktop/src/config/transformStoryTests.test.ts b/packages/agentic/storybook-desktop/src/config/transformStoryTests.test.ts index 9aceedaba9..09d5c21dfb 100644 --- a/packages/agentic/storybook-desktop/src/config/transformStoryTests.test.ts +++ b/packages/agentic/storybook-desktop/src/config/transformStoryTests.test.ts @@ -86,6 +86,16 @@ describe('native Storybook test stripping', () => { expect(output).toContain('Two'); }); + test('strips desktopIt calls and their unused import from the native story module', () => { + const output = transform(` + import { desktopIt, type WdioStory } from '@fluentui-react-native/storybook-desktop/testing'; + export const Example: WdioStory = { render: () => , wdio: [desktopIt('case', () => console.log('node-only'))] }; + `); + expect(output).not.toContain('desktopIt'); + expect(output).not.toContain('node-only'); + expect(output).toContain('render'); + }); + test.each([ [`export const Example = { wdio: importedCallback };`, /inline/], [`export const Example = { ...shared };`, /spread/], diff --git a/packages/agentic/storybook-desktop/src/config/wdio.test.ts b/packages/agentic/storybook-desktop/src/config/wdio.test.ts index dd816c2bfc..996fdf90c4 100644 --- a/packages/agentic/storybook-desktop/src/config/wdio.test.ts +++ b/packages/agentic/storybook-desktop/src/config/wdio.test.ts @@ -24,4 +24,20 @@ describe('Storybook-owned wdio configuration', () => { expect(() => resolveWdioOptions({ story: '' })).toThrow('wdio.story'); expect(() => resolveWdioOptions({ tag: ' ' })).toThrow('wdio.tag'); }); + + test('resolves an explicit input lane and rejects conflicting click configuration', () => { + const previous = process.env.DESKTOP_INPUT; + delete process.env.DESKTOP_INPUT; + try { + expect(resolveWdioOptions({ input: 'accessible' })).toMatchObject({ input: 'accessible', clickMode: 'auto' }); + expect(() => resolveWdioOptions({ input: 'physical', clickMode: 'physical' })).toThrow('cannot be combined with a click mode'); + process.env.DESKTOP_INPUT = 'physical'; + expect(resolveWdioOptions().input).toBe('physical'); + expect(resolveWdioOptions({ input: 'accessible' }).input).toBe('accessible'); + expect(() => resolveWdioOptions({ clickMode: 'physical' })).toThrow('cannot be combined'); + } finally { + if (previous === undefined) delete process.env.DESKTOP_INPUT; + else process.env.DESKTOP_INPUT = previous; + } + }); }); diff --git a/packages/agentic/storybook-desktop/src/config/wdio.ts b/packages/agentic/storybook-desktop/src/config/wdio.ts index 7e7b4102dd..f1525e685e 100644 --- a/packages/agentic/storybook-desktop/src/config/wdio.ts +++ b/packages/agentic/storybook-desktop/src/config/wdio.ts @@ -1,7 +1,9 @@ import type { DesktopWebdriverOptions } from '@fluentui-react-native/desktop-driver/wdio'; +import type { DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; export type DesktopStorybookWdioOptions = { clickMode?: DesktopWebdriverOptions['clickMode']; + input?: DesktopInputLane; reporter?: 'spec' | 'tap' | 'dot'; story?: string; tag?: string; @@ -13,6 +15,15 @@ export function resolveWdioOptions( options: DesktopStorybookWdioOptions = {}, ): Readonly> & DesktopStorybookWdioOptions> { const { timeoutMs = 30_000, reporter = 'spec', clickMode = 'auto' } = options; + const requestedInput = options.input ?? process.env.DESKTOP_INPUT; + let input: DesktopInputLane | undefined; + if (requestedInput !== undefined) { + if (!isDesktopInputLane(requestedInput)) throw new TypeError('wdio.input must be accessible or physical.'); + input = requestedInput; + } + if (input && clickMode !== 'auto') { + throw new TypeError('wdio.input cannot be combined with a click mode other than auto.'); + } if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 2_147_483_647) { throw new TypeError('wdio.timeoutMs must be a positive 32-bit integer.'); } @@ -27,5 +38,15 @@ export function resolveWdioOptions( throw new TypeError(`wdio.${key} must be a non-empty string.`); } } - return Object.freeze({ ...options, timeoutMs, reporter, clickMode }); + return Object.freeze({ + ...options, + ...(input ? { input } : {}), + timeoutMs, + reporter, + clickMode, + }); +} + +function isDesktopInputLane(value: string): value is DesktopInputLane { + return value === 'accessible' || value === 'physical'; } diff --git a/packages/agentic/storybook-desktop/src/driver/driverManifest.ts b/packages/agentic/storybook-desktop/src/driver/driverManifest.ts index 6b5aa6afcc..69dd15764d 100644 --- a/packages/agentic/storybook-desktop/src/driver/driverManifest.ts +++ b/packages/agentic/storybook-desktop/src/driver/driverManifest.ts @@ -59,6 +59,7 @@ export type CreateDesktopStorybookDriverManifestOptions = { platform: Platforms; renderer?: DesktopStorybookDriverManifest['renderer']; storyManifest: DesktopStoryManifest; + wdio?: DesktopStorybookWdioOptions; }; /** Returns the WebDriver target ID the embedded Storybook driver registers for an endpoint and renderer. */ @@ -81,6 +82,7 @@ export function createDesktopStorybookDriverManifest({ platform, renderer = platform === 'win32' ? 'paper' : 'fabric', storyManifest, + wdio, }: CreateDesktopStorybookDriverManifestOptions): DesktopStorybookDriverManifest { const nativeOptions = config.getNativeDriverOptions(platform); if (nativeOptions === false) { @@ -121,7 +123,7 @@ export function createDesktopStorybookDriverManifest({ storybookPort: instance.storybookPort, targetId: desktopStorybookTargetId(config, platform, renderer), testIDPrefix: config.testIDPrefix, - wdio: config.wdio, + wdio: wdio ?? config.wdio, }); } diff --git a/packages/agentic/storybook-desktop/src/driver/storyManifest.ts b/packages/agentic/storybook-desktop/src/driver/storyManifest.ts index c1f0a039f8..837bcf82e2 100644 --- a/packages/agentic/storybook-desktop/src/driver/storyManifest.ts +++ b/packages/agentic/storybook-desktop/src/driver/storyManifest.ts @@ -86,6 +86,7 @@ export async function createDesktopStoryManifest( for (const [exportName, staticStory] of stories) { const wdio = wdioTests.get(exportName); const testNames = wdio?.tests.flatMap(({ name }) => (name === undefined ? [] : [name])); + const cases = wdio?.tests.flatMap(({ name, options }) => (name === undefined || options === undefined ? [] : [{ name, options }])); const tests = readDesktopStoryTests( extractDesktopStoryTests(csf._storyAnnotations?.[exportName]?.parameters, sourceFile, staticStory.id) ?? staticStory.parameters?.desktopDriver, @@ -100,7 +101,16 @@ export async function createDesktopStoryManifest( tags: [...new Set([...(csf.meta?.tags ?? []), ...(staticStory.tags ?? []), 'story'])].sort(), title: csf.meta?.title ?? staticStory.id.split('--')[0], ...(tests ? { tests } : {}), - ...(wdio ? { wdio: { digest: wdio.digest, exportName, ...(testNames?.length ? { testNames } : {}) } } : {}), + ...(wdio + ? { + wdio: { + digest: wdio.digest, + exportName, + ...(testNames?.length ? { testNames } : {}), + ...(cases?.length ? { cases } : {}), + }, + } + : {}), }); } diff --git a/packages/agentic/storybook-desktop/src/testing/desktopIt.ts b/packages/agentic/storybook-desktop/src/testing/desktopIt.ts new file mode 100644 index 0000000000..c95a7580eb --- /dev/null +++ b/packages/agentic/storybook-desktop/src/testing/desktopIt.ts @@ -0,0 +1,11 @@ +import type { WdioDescriptor, WdioStoryCase, WdioStoryTest } from './types.js'; + +export function desktopIt(name: string, run: WdioStoryTest): WdioStoryCase; +export function desktopIt(name: string, options: Omit, run: WdioStoryTest): WdioStoryCase; +export function desktopIt(name: string, optionsOrRun: Omit | WdioStoryTest, run?: WdioStoryTest): WdioStoryCase { + if (!name.trim()) throw new TypeError('A desktop story test requires a non-empty name.'); + const options = typeof optionsOrRun === 'function' ? {} : optionsOrRun; + const callback = typeof optionsOrRun === 'function' ? optionsOrRun : run; + if (typeof callback !== 'function') throw new TypeError('A desktop story test requires a callback.'); + return { name, ...options, run: callback }; +} diff --git a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts index f73abf0e39..0a5ff18bb4 100644 --- a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts +++ b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts @@ -33,6 +33,23 @@ describe('extractWdioTests', () => { ); }); + test('extracts desktopIt calls and their static descriptors without executing the story', () => { + const result = extractWdioTests( + ` + import { desktopIt as test } from '@fluentui-react-native/storybook-desktop/testing'; + export const Example = { wdio: [ + test('accessible action', { requires: ['click'], platforms: ['!win32'] }, async ({ browser }) => { await browser.$('~button').click(); }), + test('default case', async () => {}), + ] }; + `, + sourceFile, + ).get('Example')!; + expect(result.tests.map(({ name, options }) => ({ name, input: options?.input, requires: options?.requires }))).toEqual([ + { name: 'accessible action', input: 'both', requires: ['click'] }, + { name: 'default case', input: 'both', requires: undefined }, + ]); + }); + test.each([ ['{}', /at least one/], ["{ '': () => {} }", /non-empty/], @@ -46,6 +63,20 @@ describe('extractWdioTests', () => { expect(() => extractWdioTests(`export const Example = { wdio: ${value} };`, sourceFile)).toThrow(message); }); + test.each([ + [`export const Example = { wdio: [desktopIt('case', () => {})] };`, /imported from/], + [ + `import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; export const Example = { wdio: [desktopIt('case', { input: mode }, () => {})] };`, + /strict JSON literal/, + ], + [ + `import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; const name = 'case'; export const Example = { wdio: [desktopIt(name, () => {})] };`, + /literal test name/, + ], + ])('rejects non-static desktopIt descriptors', (source, message) => { + expect(() => extractWdioTests(source, sourceFile)).toThrow(message); + }); + test('extracts a standalone typed callback without loading the native story module', async () => { const tests = extractWdioTests( ` diff --git a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts index 493d5239bf..58e2eaf30f 100644 --- a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts +++ b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts @@ -1,6 +1,8 @@ import { createHash } from 'node:crypto'; import { transformSync, type NodePath, type types } from '@babel/core'; +import { validateDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; +import type { ResolvedDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; import storyTests from '../../config/transform-story-tests.cjs'; @@ -8,7 +10,7 @@ const { collectWdioTests, compileWdioTest } = storyTests; export type ExtractedWdioStory = { digest: string; - tests: readonly { name?: string; code: string }[]; + tests: readonly { name?: string; code: string; options?: ResolvedDesktopCaseOptions }[]; }; export function extractWdioTests(code: string, sourceFile: string): ReadonlyMap { @@ -24,9 +26,10 @@ export function extractWdioTests(code: string, sourceFile: string): ReadonlyMap< visitor: { Program(program: NodePath) { for (const [exportName, { callbacks, value }] of collectWdioTests(program)) { - const compiled = callbacks.map(({ callback, name }) => ({ + const compiled = callbacks.map(({ callback, name, options }) => ({ ...(name === undefined ? {} : { name }), code: compileWdioTest(callback, sourceFile, code), + ...(options !== undefined ? { options: validateDesktopCaseOptions(options, `${sourceFile}.wdio`) } : {}), })); const source = code.slice(value.node.start!, value.node.end!); tests.set(exportName, { diff --git a/packages/agentic/storybook-desktop/src/testing/index.ts b/packages/agentic/storybook-desktop/src/testing/index.ts index e5df94e7d3..b1b449f558 100644 --- a/packages/agentic/storybook-desktop/src/testing/index.ts +++ b/packages/agentic/storybook-desktop/src/testing/index.ts @@ -1 +1,2 @@ -export type { WdioStory, WdioStoryContext, WdioStoryTest, WdioStoryTests } from './types.js'; +export { desktopIt } from './desktopIt.js'; +export type { WdioDescriptor, WdioStory, WdioStoryCase, WdioStoryContext, WdioStoryTest, WdioStoryTests } from './types.js'; diff --git a/packages/agentic/storybook-desktop/src/testing/runWdioTests.test.ts b/packages/agentic/storybook-desktop/src/testing/runWdioTests.test.ts new file mode 100644 index 0000000000..5860dd8e4f --- /dev/null +++ b/packages/agentic/storybook-desktop/src/testing/runWdioTests.test.ts @@ -0,0 +1,68 @@ +import { validateDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopStoryManifest } from '@fluentui-react-native/desktop-driver'; + +import { missingRequirements, resolveCaseClickMode, selectWdioCases, selectWdioStories } from './runWdioTests.js'; + +describe('WDIO input lane selection', () => { + const manifest: DesktopStoryManifest = { + endpoint: 'win32', + entries: [ + { + id: 'button', + name: 'Button', + packageName: 'components', + sourcePath: 'button.stories.tsx', + tags: ['story'], + title: 'Button', + wdio: { + digest: 'digest', + exportName: 'Default', + testNames: ['adaptive', 'accessible', 'win32-only'], + cases: [ + { name: 'adaptive', options: validateDesktopCaseOptions({ requires: ['click'] }) }, + { name: 'accessible', options: validateDesktopCaseOptions({ requires: ['accessibility-click'] }) }, + { name: 'win32-only', options: validateDesktopCaseOptions({ platforms: ['win32'] }) }, + ], + }, + }, + ], + platformManifestDigest: 'digest', + portablePlanDigest: 'digest', + schemaVersion: 1, + }; + + test('selects by modality and target platform', () => { + expect(selectWdioStories(manifest, { input: 'accessible' }, 'win32').map(({ id }) => id)).toEqual(['button']); + expect(selectWdioCases(manifest.entries[0], { input: 'accessible' }, 'win32')?.map(({ name }) => name)).toEqual([ + 'adaptive', + 'accessible', + 'win32-only', + ]); + expect(selectWdioCases(manifest.entries[0], {}, 'macos')?.map(({ name }) => name)).toEqual(['adaptive', 'accessible']); + expect(selectWdioCases(manifest.entries[0], { input: 'physical' }, 'win32')?.map(({ name }) => name)).toEqual([ + 'adaptive', + 'win32-only', + ]); + expect(selectWdioCases(manifest.entries[0], { input: 'physical', test: 'accessible' }, 'macos')).toEqual([]); + expect(selectWdioCases(manifest.entries[0], { input: 'physical', test: 'win32-only' }, 'macos')).toEqual([]); + }); + + test('rejects legacy callback entries in a lane', () => { + const legacy = { + ...manifest, + entries: [{ ...manifest.entries[0], wdio: { digest: 'legacy', exportName: 'Default', testNames: ['legacy'] } }], + } satisfies DesktopStoryManifest; + expect(() => selectWdioStories(legacy, { input: 'physical' }, 'win32')).toThrow('declare it with desktopIt'); + }); + + test('adapts click requirements to each lane without fallback', () => { + const descriptor = validateDesktopCaseOptions({ requires: ['click'] }); + const accessibleMode = resolveCaseClickMode(descriptor, 'auto', 'accessible'); + const physicalMode = resolveCaseClickMode(descriptor, 'auto', 'physical'); + expect(accessibleMode).toBe('accessibility'); + expect(physicalMode).toBe('physical'); + expect(missingRequirements(descriptor, { accessibilityClick: true, physicalClick: false }, physicalMode)).toEqual(['click']); + expect(missingRequirements(descriptor, { accessibilityClick: false, physicalClick: true }, accessibleMode)).toEqual(['click']); + expect(missingRequirements(descriptor, { accessibilityClick: true }, accessibleMode)).toEqual([]); + }); +}); diff --git a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts index 52cacc56c1..8ed1c3a371 100644 --- a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts +++ b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts @@ -4,7 +4,8 @@ import path from 'node:path'; import { pathToFileURL } from 'node:url'; import type { DesktopStoryManifest, DesktopStoryManifestEntry } from '@fluentui-react-native/desktop-driver'; -import type { DesktopArtifact } from '@fluentui-react-native/desktop-driver/authoring'; +import { matchesDesktopPlatforms } from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopArtifact, DesktopInputLane, ResolvedDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; import { ArtifactManager, captureDesktopFailureEvidence } from '@fluentui-react-native/desktop-driver/artifacts'; import { connectDesktopWebdriver } from '@fluentui-react-native/desktop-driver/wdio'; import type { DesktopReportWriter } from '@fluentui-react-native/desktop-test/reporting'; @@ -21,10 +22,11 @@ import { formatDesktopStorybookError, writeDesktopStorybookFailure, type Desktop export type WdioStoryTestResult = { storyId: string; testName?: string; - status: 'passed' | 'failed' | 'skipped'; + status: 'passed' | 'failed' | 'skipped' | 'unsupported'; durationMs: number; error?: string; skipReason?: string; + gap?: 'unsupported' | 'quarantine'; evidenceErrors?: string[]; artifacts?: readonly DesktopArtifact[]; }; @@ -53,23 +55,50 @@ export class WdioStoryTestRunError extends Error { } } +class MissingRequirementsError extends Error { + constructor(requirements: readonly string[]) { + super(`Missing desktop requirements: ${requirements.join(', ')}.`); + } +} + export function selectWdioStories( manifest: DesktopStoryManifest, options: DesktopStorybookWdioOptions, + platform?: Platforms, ): readonly DesktopStoryManifestEntry[] { return manifest.entries .filter((entry) => entry.wdio && (!options.story || path.matchesGlob(entry.id, options.story))) .filter((entry) => !options.tag || entry.tags.includes(options.tag)) - .filter((entry) => (entry.wdio?.testNames ?? ['default']).some((name) => matchesWdioTest(name, options.test))) + .filter((entry) => { + if (options.input || entry.wdio?.cases) return (selectWdioCases(entry, options, platform)?.length ?? 0) > 0; + return (entry.wdio?.testNames ?? ['default']).some((name) => matchesWdioTest(name, options.test)); + }) .sort((left, right) => left.id.localeCompare(right.id)); } +export function selectWdioCases( + entry: DesktopStoryManifestEntry, + options: DesktopStorybookWdioOptions, + platform?: Platforms, +): NonNullable['cases'] { + if (!entry.wdio?.cases?.length) { + if (!options.input) return undefined; + throw new Error(`Input lane "${options.input}" cannot run legacy wdio callback in "${entry.id}"; declare it with desktopIt.`); + } + return entry.wdio.cases.filter( + ({ name, options: descriptor }) => + matchesWdioTest(name, options.test) && + (!platform || matchesDesktopPlatforms(descriptor.platforms, platform)) && + (!options.input || descriptor.input === 'both' || descriptor.input === options.input), + ); +} + export async function runWdioStoryTests( options: RunWdioStoryTestsOptions, runner?: DesktopCommandRunner, ): Promise { const settings = resolveWdioOptions(options); - const stories = selectWdioStories(options.manifest, settings); + const stories = selectWdioStories(options.manifest, settings, options.platform); if (stories.length === 0) { throw new Error('No executable wdio story tests matched the requested selection.'); } @@ -79,9 +108,19 @@ export async function runWdioStoryTests( if (!extracted || extracted.digest !== entry.wdio!.digest) { throw new Error(`Executable test source changed for ${entry.id}; regenerate the Storybook manifest.`); } + const selectedCases = selectWdioCases(entry, settings, options.platform); return extracted.tests .filter(({ name }) => matchesWdioTest(name ?? 'default', settings.test)) - .map(({ name, code }) => ({ entry, code, testName: name })); + .map(({ name, code }) => ({ + entry, + code, + testName: name, + descriptor: entry.wdio?.cases?.find((testCase) => testCase.name === name)?.options, + })) + .filter(({ testName }) => { + if (!selectedCases) return !settings.input; + return selectedCases.some(({ name }) => name === testName); + }); }); if (cases.length === 0) throw new Error('No executable wdio tests matched the requested selection.'); const artifacts = new ArtifactManager( @@ -112,7 +151,7 @@ export async function runWdioStoryTests( try { fs.mkdirSync(generatedRoot, { recursive: true }); directory = fs.mkdtempSync(path.join(generatedRoot, 'run-')); - for (const [index, { entry, code, testName }] of cases.entries()) { + for (const [index, { entry, code, testName, descriptor }] of cases.entries()) { options.signal?.throwIfAborted(); const displayName = testName === undefined ? entry.id : `${entry.id} / ${testName}`; const artifactId = @@ -132,9 +171,29 @@ export async function runWdioStoryTests( const context = { story: entry.id, test: testName ?? 'default', endpoint: options.platform }; options.report?.emit({ kind: 'scope-start', payload: { label: displayName, category: 'worker' } }, testId, context, testId); options.report?.emit({ kind: 'test-start', payload: { testId, title: displayName, attempt: 0 } }, testId, context); + const gap = descriptor && declaredGapFor(descriptor, options.platform, settings.input); + if (gap) { + result.status = 'unsupported'; + result.skipReason = gap.reason; + result.gap = gap.kind; + result.durationMs = Date.now() - started; + results.push(result); + options.report?.emit( + { + kind: 'test-result', + payload: { testId, title: displayName, attempt: 0, status: 'unsupported', durationMs: result.durationMs, reason: gap.reason }, + }, + testId, + context, + ); + options.report?.emit({ kind: 'scope-end', payload: { status: 'unsupported' } }, testId); + activeResult = undefined; + continue; + } + const clickMode = resolveCaseClickMode(descriptor, settings.clickMode, settings.input); // The supervisor owns the session, so even a crashed or timed-out Node worker cannot strand it. const desktop = await connectDesktopWebdriver({ - clickMode: settings.clickMode, + clickMode, launchMode: 'attach', platformName: options.platform === 'macos' ? 'macos' : 'windows', targetId: options.targetId, @@ -147,6 +206,8 @@ export async function runWdioStoryTests( if (live.endpoint !== options.platform || live.platformManifestDigest !== options.manifest.platformManifestDigest) { throw new Error('The running Storybook manifest is stale or targets a different platform. Restart storybook driver and the app.'); } + const missing = missingRequirements(descriptor, desktop.browser.capabilities['furn:features'], clickMode); + if (missing.length) throw new MissingRequirementsError(missing); phase = 'navigation'; await desktop.openStory(entry.id); const resultPath = path.join(directory, `${index}.json`); @@ -199,27 +260,32 @@ export async function runWdioStoryTests( result.skipReason = outcome.skipReason; } } catch (error) { - failure = error; - executionFailures.push(error); - result.error = formatDesktopStorybookError(error); - writeDesktopStorybookFailure(`wdio "${displayName}" during ${phase}`, error, options.errorOutput); - const evidence = await captureDesktopFailureEvidence(desktop.session, artifacts, artifactId); - result.artifacts = evidence.artifacts; - result.evidenceErrors = evidence.errors; - for (const artifact of result.artifacts) - options.report?.emit( - { - kind: 'artifact', - payload: { - path: path.relative(options.report.root, path.join(artifacts.root, artifact.path)), - description: `${displayName}: ${artifact.kind}`, + if (error instanceof MissingRequirementsError) { + result.status = 'unsupported'; + result.skipReason = error.message; + } else { + failure = error; + executionFailures.push(error); + result.error = formatDesktopStorybookError(error); + writeDesktopStorybookFailure(`wdio "${displayName}" during ${phase}`, error, options.errorOutput); + const evidence = await captureDesktopFailureEvidence(desktop.session, artifacts, artifactId); + result.artifacts = evidence.artifacts; + result.evidenceErrors = evidence.errors; + for (const artifact of result.artifacts) + options.report?.emit( + { + kind: 'artifact', + payload: { + path: path.relative(options.report.root, path.join(artifacts.root, artifact.path)), + description: `${displayName}: ${artifact.kind}`, + }, }, - }, - testId, - ); - for (const evidenceError of result.evidenceErrors) { - options.report?.fail(new Error(evidenceError), 'evidence', testId); - writeDesktopStorybookFailure(`evidence for "${displayName}"`, evidenceError, options.errorOutput); + testId, + ); + for (const evidenceError of result.evidenceErrors) { + options.report?.fail(new Error(evidenceError), 'evidence', testId); + writeDesktopStorybookFailure(`evidence for "${displayName}"`, evidenceError, options.errorOutput); + } } } phase = 'session cleanup'; @@ -243,7 +309,7 @@ export async function runWdioStoryTests( attempt: 0, status: result.status, durationMs: result.durationMs, - ...(result.status === 'skipped' ? { reason: result.skipReason } : {}), + ...(['skipped', 'unsupported'].includes(result.status) ? { reason: result.skipReason } : {}), ...(failure !== undefined ? { failure: options.report.failure(failure, 'test') } : {}), }, }, @@ -283,7 +349,11 @@ export async function runWdioStoryTests( const failures: unknown[] = infrastructureFailure === undefined ? [] : [infrastructureFailure]; failures.push(...executionFailures); const allSkipped = results.every(({ status }) => status === 'skipped'); - const incomplete = results.length !== cases.length || (allSkipped && !options.optional); + const laneIncomplete = + !!settings.input && + (!results.some(({ status }) => status === 'passed' || status === 'failed') || + results.some(({ status, gap }) => status === 'skipped' || (status === 'unsupported' && gap === undefined))); + const incomplete = results.length !== cases.length || (allSkipped && !options.optional) || laneIncomplete; if (incomplete && failures.length === 0) failures.push(new Error('Required executable story cohort did not execute any tests or left selected work unfinished.')); try { @@ -347,6 +417,75 @@ function csfTestId(storyId: string, testName?: string): string { .digest('hex')}`; } +export function resolveCaseClickMode( + descriptor: ResolvedDesktopCaseOptions | undefined, + configured: 'auto' | 'physical' | 'accessibility', + input: DesktopInputLane | undefined, +): 'auto' | 'physical' | 'accessibility' { + if (input === 'accessible') return 'accessibility'; + if (input === 'physical') return 'physical'; + if (descriptor?.input === 'accessible' || descriptor?.requires?.includes('accessibility-click')) return 'accessibility'; + if (descriptor?.input === 'physical') return 'physical'; + return configured; +} + +export function missingRequirements( + descriptor: ResolvedDesktopCaseOptions | undefined, + features: Readonly> | undefined, + clickMode: 'auto' | 'physical' | 'accessibility', +): string[] { + if (!descriptor?.requires?.length) return []; + const missing: string[] = []; + for (const requirement of descriptor.requires) { + const feature = + requirement === 'click' + ? clickMode === 'physical' + ? 'physicalClick' + : clickMode === 'accessibility' + ? 'accessibilityClick' + : undefined + : requirement === 'physical-click' + ? 'physicalClick' + : requirement === 'accessibility-click' + ? 'accessibilityClick' + : requirement === 'accessibility-actions' + ? 'accessibilityActions' + : requirement === 'element-screenshot' + ? 'elementScreenshot' + : requirement === 'set-window-rect' + ? 'setWindowRect' + : requirement; + if (requirement === 'click' && clickMode === 'auto') { + if (!features?.physicalClick && !features?.accessibilityClick) missing.push(requirement); + } else if (feature && features?.[feature] !== true) { + missing.push(requirement); + } + } + return missing; +} + +function declaredGapFor( + descriptor: ResolvedDesktopCaseOptions, + endpoint: Platforms, + input: DesktopInputLane | undefined, +): { kind: 'unsupported' | 'quarantine'; reason: string } | undefined { + if (descriptor.unsupported && matchesDesktopPlatforms(descriptor.unsupported.platforms, endpoint)) { + return { kind: 'unsupported', reason: descriptor.unsupported.reason }; + } + const quarantine = descriptor.quarantine; + if ( + quarantine && + (!quarantine.platforms || matchesDesktopPlatforms(quarantine.platforms, endpoint)) && + (!quarantine.input || quarantine.input === input) + ) { + if (Date.parse(`${quarantine.expires}T00:00:00Z`) < Date.now()) { + throw new Error(`Quarantine ${quarantine.issue} expired on ${quarantine.expires}.`); + } + return { kind: 'quarantine', reason: `${quarantine.issue}: ${quarantine.reason}` }; + } + return undefined; +} + function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } diff --git a/packages/agentic/storybook-desktop/src/testing/types.ts b/packages/agentic/storybook-desktop/src/testing/types.ts index 8077303d07..d87cbfc6e1 100644 --- a/packages/agentic/storybook-desktop/src/testing/types.ts +++ b/packages/agentic/storybook-desktop/src/testing/types.ts @@ -1,6 +1,7 @@ import type { DesktopWebdriverSession } from '@fluentui-react-native/desktop-driver/wdio'; import type { DesktopEndpoint } from '@fluentui-react-native/desktop-driver'; import type { expect } from 'expect-webdriverio'; +import type { DesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; export type WdioStoryContext = { browser: DesktopWebdriverSession['browser']; @@ -13,8 +14,10 @@ export type WdioStoryContext = { export type WdioStoryTest = (context: WdioStoryContext) => void | Promise; export type WdioStoryTests = Readonly>; +export type WdioDescriptor = DesktopCaseOptions & { run: WdioStoryTest }; +export type WdioStoryCase = WdioDescriptor & { name: string }; /** Wrap StoryObj, importing this type only; callbacks execute in Node, not React Native. */ export type WdioStory = Omit & { - wdio?: WdioStoryTest | WdioStoryTests; + wdio?: readonly WdioStoryCase[] | WdioStoryTest | WdioStoryTests; }; diff --git a/packages/agentic/storybook-desktop/src/testing/wdio.types.test.ts b/packages/agentic/storybook-desktop/src/testing/wdio.types.test.ts index 10a341502e..176a4f80e6 100644 --- a/packages/agentic/storybook-desktop/src/testing/wdio.types.test.ts +++ b/packages/agentic/storybook-desktop/src/testing/wdio.types.test.ts @@ -1,4 +1,4 @@ -import type { WdioStory, WdioStoryContext } from './index.js'; +import { desktopIt, type WdioStory, type WdioStoryContext } from './index.js'; const story: WdioStory = { wdio: async ({ browser, desktop, expect, signal, skip }) => { @@ -38,6 +38,17 @@ const named: WdioStory = { }, }; +const described: WdioStory = { + wdio: [ + desktopIt('uses a lane-adaptive click', { requires: ['click'] }, async ({ browser }) => { + await (await browser.$('~button')).click(); + }), + desktopIt('has the default descriptor', async ({ browser }) => { + await browser.$('~button'); + }), + ], +}; + const wrapped: WdioStory<{ args: { disabled: boolean }; wdio?: unknown }> = { args: { disabled: false }, wdio: async ({ browser }) => { @@ -56,5 +67,6 @@ test('exposes a type-only inline callback contract', () => { expect(typeof story.wdio).toBe('function'); expect(typeof wrapped.wdio).toBe('function'); expect(typeof named.wdio).toBe('object'); + expect(Array.isArray(described.wdio)).toBe(true); expect(invalidArgs.args).toBeDefined(); }); From 451e1cb0225851bccc0b8667be03762f2463871b Mon Sep 17 00:00:00 2001 From: Jason Morse Date: Fri, 6 Nov 2026 17:40:20 -0800 Subject: [PATCH 2/2] Select and account for desktop WDIO cases by input lane Finish part 1 of the input-lane work started in f5a90cdda. Cases declare how they run with desktopIt, and a run selects the cases that apply to its input lane, platform and scope and accounts for everything it excludes. desktop-driver - Add pure case selection and click-mode resolution, and move the requirement preflight and click-mode helpers here. - Add the authoring/input leaf export so config-loading code can import scope helpers from source without the authoring index. desktop-test - Report the lane: a lane run is incomplete when it has skipped or undeclared-unsupported results, an empty selection or nothing executed. - Add the Jasmine adapter (desktop-test/jasmine) with desktopIt and desktopDescribe, plus config-level test scopes and --input. storybook-desktop - Add --input accessible|physical to storybook test, driver and smoke. The lane comes from the flag, DESKTOP_INPUT or the attached driver. - Select cases before they run, resolve declared gaps, assess each cohort, and reject direct input calls in declared cases at extraction. - Fail closed when an accessible lane attaches to a driver that does not mask keyboard, physical click and wheel input. native-core and apps/E2E - Migrate the native-core Callout and FocusZone tests and ButtonV1.spec.win.ts to desktopIt. Update the package references and add patch changesets. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .changeset/input-lanes.md | 3 +- .changeset/native-core-desktop-it.md | 5 + .../src/ButtonV1/specs/ButtonV1.spec.win.ts | 26 +- packages/agentic/desktop-driver/README.md | 35 +- packages/agentic/desktop-driver/package.json | 5 + .../desktop-driver/references/architecture.md | 38 +- .../desktop-driver/references/contributing.md | 35 +- .../references/test-integration.md | 27 +- .../desktop-driver/src/authoring/index.ts | 26 +- .../src/authoring/input.test.ts | 125 +++++- .../desktop-driver/src/authoring/input.ts | 172 +++++++- .../src/authoring/selection.test.ts | 96 +++++ .../desktop-driver/src/authoring/selection.ts | 116 ++++++ packages/agentic/desktop-driver/src/index.ts | 26 +- packages/agentic/desktop-test/package.json | 5 + .../desktop-test/references/commands.md | 35 +- .../desktop-test/references/configuration.md | 28 ++ .../desktop-test/references/contributing.md | 13 +- .../desktop-test/references/reporting.md | 55 ++- .../desktop-test/references/wdio-suites.md | 65 +++ packages/agentic/desktop-test/src/cli.ts | 30 +- packages/agentic/desktop-test/src/config.ts | 67 ++- .../desktop-test/src/contracts.contract.cjs | 236 ++++++++++- .../desktop-test/src/contracts.test.ts | 4 + packages/agentic/desktop-test/src/index.ts | 2 +- .../agentic/desktop-test/src/integration.ts | 3 + .../desktop-test/src/jasmine/adapter.test.ts | 208 ++++++++++ .../desktop-test/src/jasmine/adapter.ts | 103 +++++ .../agentic/desktop-test/src/jasmine/index.ts | 42 ++ .../desktop-test/src/jasmine/registry.ts | 72 ++++ .../src/jasmine/selection.test.ts | 50 +++ .../desktop-test/src/jasmine/selection.ts | 59 +++ .../src/jasmine/specSelection.test.ts | 183 +++++++++ .../desktop-test/src/jasmine/specSelection.ts | 96 +++++ .../src/reporting/collector.test.ts | 121 ++++++ .../desktop-test/src/reporting/collector.ts | 32 +- .../desktop-test/src/reporting/index.ts | 26 +- .../desktop-test/src/reporting/lane.test.ts | 34 ++ .../desktop-test/src/reporting/lane.ts | 37 ++ .../src/reporting/protocol.test.ts | 50 ++- .../desktop-test/src/reporting/protocol.ts | 22 +- .../src/reporting/results.test.ts | 169 +++++++- .../desktop-test/src/reporting/results.ts | 110 ++++- .../src/reporting/runQueries.test.ts | 89 ++++ .../desktop-test/src/reporting/runQueries.ts | 56 ++- .../src/reporting/structured.test.ts | 18 + .../desktop-test/src/reporting/structured.ts | 5 + .../desktop-test/src/supervisor.test.ts | 71 ++++ .../agentic/desktop-test/src/supervisor.ts | 8 + .../desktop-test/src/wdio/configuration.ts | 12 +- .../agentic/desktop-test/src/wdio/reporter.ts | 28 +- packages/agentic/desktop-test/src/wdio/run.ts | 30 +- .../native-core/references/contributing.md | 9 + .../components/callout/Callout.stories.tsx | 383 +++++++++--------- .../callout/menuComposition.wdio.ts | 12 +- .../focus-zone/FocusZone.stories.tsx | 97 ++--- .../src/legacy/callout/Callout.stories.tsx | 75 ++-- .../storybook-desktop/config/server.cjs | 7 + .../config/storybook-control.cjs | 6 +- .../config/transform-story-tests.cjs | 131 ++++++ .../config/transform-story-tests.d.cts | 4 + .../storybook-desktop/references/commands.md | 16 +- .../references/contributing.md | 10 + .../references/story-tests.md | 206 +++++++--- .../src/cli/DesktopStorybookCli.test.ts | 144 ++++++- .../src/cli/DesktopStorybookCli.ts | 83 ++-- .../src/cli/createDesktopStorybookCommand.ts | 18 + .../src/cli/inputLane.test.ts | 101 +++++ .../storybook-desktop/src/cli/inputLane.ts | 42 ++ .../src/cli/smokeTests.test.ts | 124 ++++++ .../storybook-desktop/src/cli/smokeTests.ts | 37 +- .../src/config/integration.ts | 4 +- .../src/config/makeDesktopStorybookConfig.ts | 11 +- .../storybook-desktop/src/config/wdio.test.ts | 5 + .../src/driver/driverManifest.ts | 24 ++ .../src/driver/serverIntegration.test.ts | 58 +++ .../src/testing/extractWdioTests.test.ts | 111 +++++ .../src/testing/extractWdioTests.ts | 19 +- .../src/testing/laneVerdict.test.ts | 75 ++++ .../src/testing/laneVerdict.ts | 72 ++++ .../src/testing/runWdioTests.test.ts | 68 ---- .../src/testing/runWdioTests.ts | 224 ++++------ .../src/testing/selection.test.ts | 301 ++++++++++++++ .../src/testing/selection.ts | 194 +++++++++ .../src/testing/wdioRunner.contract.cjs | 64 +++ .../src/testing/wdioRunner.test.ts | 25 ++ 86 files changed, 5007 insertions(+), 762 deletions(-) create mode 100644 .changeset/native-core-desktop-it.md create mode 100644 packages/agentic/desktop-driver/src/authoring/selection.test.ts create mode 100644 packages/agentic/desktop-driver/src/authoring/selection.ts create mode 100644 packages/agentic/desktop-test/src/jasmine/adapter.test.ts create mode 100644 packages/agentic/desktop-test/src/jasmine/adapter.ts create mode 100644 packages/agentic/desktop-test/src/jasmine/index.ts create mode 100644 packages/agentic/desktop-test/src/jasmine/registry.ts create mode 100644 packages/agentic/desktop-test/src/jasmine/selection.test.ts create mode 100644 packages/agentic/desktop-test/src/jasmine/selection.ts create mode 100644 packages/agentic/desktop-test/src/jasmine/specSelection.test.ts create mode 100644 packages/agentic/desktop-test/src/jasmine/specSelection.ts create mode 100644 packages/agentic/desktop-test/src/reporting/lane.test.ts create mode 100644 packages/agentic/desktop-test/src/reporting/lane.ts create mode 100644 packages/agentic/desktop-test/src/supervisor.test.ts create mode 100644 packages/agentic/storybook-desktop/src/cli/inputLane.test.ts create mode 100644 packages/agentic/storybook-desktop/src/cli/inputLane.ts create mode 100644 packages/agentic/storybook-desktop/src/testing/laneVerdict.test.ts create mode 100644 packages/agentic/storybook-desktop/src/testing/laneVerdict.ts delete mode 100644 packages/agentic/storybook-desktop/src/testing/runWdioTests.test.ts create mode 100644 packages/agentic/storybook-desktop/src/testing/selection.test.ts create mode 100644 packages/agentic/storybook-desktop/src/testing/selection.ts diff --git a/.changeset/input-lanes.md b/.changeset/input-lanes.md index 4c06b6b268..0b98bd6d1c 100644 --- a/.changeset/input-lanes.md +++ b/.changeset/input-lanes.md @@ -1,6 +1,7 @@ --- '@fluentui-react-native/desktop-driver': patch +'@fluentui-react-native/desktop-test': patch '@fluentui-react-native/storybook-desktop': patch --- -Add modality-aware desktop test descriptors and Storybook input-lane selection. +Add modality-aware desktop test descriptors: `desktopIt` and `desktopDescribe` declare `requires`, `input`, `platforms`, `scope`, and known gaps for Storybook cases and, through `desktop-test/jasmine`, WebdriverIO Jasmine specs. `--input accessible|physical` (or `DESKTOP_INPUT`) runs one input lane, selects the cases that apply, masks the host for the accessible lane, and reports the lane and what it excluded. diff --git a/.changeset/native-core-desktop-it.md b/.changeset/native-core-desktop-it.md new file mode 100644 index 0000000000..9195e49344 --- /dev/null +++ b/.changeset/native-core-desktop-it.md @@ -0,0 +1,5 @@ +--- +'@fluentui-react-native/native-core': patch +--- + +Declare the Callout and FocusZone desktop tests with `desktopIt` and `requires`, so each runs in the input lane it needs and the Win32 Tab-exit gap is declared instead of skipped at run time. diff --git a/apps/E2E/src/ButtonV1/specs/ButtonV1.spec.win.ts b/apps/E2E/src/ButtonV1/specs/ButtonV1.spec.win.ts index 2d03c4cb2d..4542acf99f 100644 --- a/apps/E2E/src/ButtonV1/specs/ButtonV1.spec.win.ts +++ b/apps/E2E/src/ButtonV1/specs/ButtonV1.spec.win.ts @@ -1,14 +1,17 @@ +import { desktopDescribe, desktopIt } from '@fluentui-react-native/desktop-test/jasmine'; + import { BUTTON_ACCESSIBILITY_LABEL, BUTTON_TEST_COMPONENT_LABEL } from '../../ButtonLegacy/consts'; import { BUTTON_A11Y_ROLE, Keys, Attribute } from '../../common/consts'; import ButtonV1PageObject from '../pages/ButtonV1PageObject'; // Before testing begins, allow up to 60 seconds for app to open -describe('ButtonV1 Testing Initialization', () => { - it('Wait for app load', async () => { +desktopDescribe('ButtonV1 Testing Initialization', () => { + desktopIt('Wait for app load', async () => { expect(await ButtonV1PageObject.waitForInitialPageToDisplay()).toBeTrue(); }); - it('Click and navigate to ButtonV1 test page', async () => { + // Navigation scrolls the page list with the wheel whenever the page button is out of view. + desktopIt('Click and navigate to ButtonV1 test page', { requires: ['click', 'wheel'] }, async () => { /* Click on component button to navigate to test page */ expect(await ButtonV1PageObject.navigateToPageAndLoadTests()).toBeTrue(); @@ -21,18 +24,19 @@ describe('ButtonV1 Testing Initialization', () => { }); }); -describe('ButtonV1 Accessibility Testing', () => { +// The setup scrolls the test element into view, which can inject native wheel input, so this group is physical. +desktopDescribe('ButtonV1 Accessibility Testing', { requires: ['wheel'] }, () => { beforeEach(async () => { await ButtonV1PageObject.scrollToTestElement(); }); - it('Validate "accessibilityRole" value defaults to "ControlType.Button".', async () => { + desktopIt('Validate "accessibilityRole" value defaults to "ControlType.Button".', async () => { await expect( await ButtonV1PageObject.compareAttribute(ButtonV1PageObject._primaryComponent, Attribute.AccessibilityRole, BUTTON_A11Y_ROLE), ).toBeTruthy(); }); - it('Set "accessibilityLabel" prop. Validate "accessibilityLabel" value propagates to "Name" element attribute.', async () => { + desktopIt('Set "accessibilityLabel" prop. Validate "accessibilityLabel" value propagates to "Name" element attribute.', async () => { await expect( await ButtonV1PageObject.compareAttribute( ButtonV1PageObject._primaryComponent, @@ -42,7 +46,7 @@ describe('ButtonV1 Accessibility Testing', () => { ).toBeTruthy(); }); - it('Do NOT set "accessibilityLabel" prop. Validate "Name" element attribute defaults to the button label.', async () => { + desktopIt('Do NOT set "accessibilityLabel" prop. Validate "Name" element attribute defaults to the button label.', async () => { await expect( await ButtonV1PageObject.compareAttribute( ButtonV1PageObject._secondaryComponent, @@ -53,7 +57,7 @@ describe('ButtonV1 Accessibility Testing', () => { }); }); -describe('ButtonV1 Functional Testing', () => { +desktopDescribe('ButtonV1 Functional Testing', { requires: ['wheel'] }, () => { /* Scrolls and waits for the Button to be visible on the Test Page */ beforeEach(async () => { await ButtonV1PageObject.scrollToTestElement(); @@ -63,7 +67,7 @@ describe('ButtonV1 Functional Testing', () => { await ButtonV1PageObject.resetTest(); }); - it('Click primary button. Validate onClick() callback was fired.', async () => { + desktopIt('Click primary button. Validate onClick() callback was fired.', { requires: ['physical-click'] }, async () => { await ButtonV1PageObject.click(ButtonV1PageObject._primaryComponent); await expect( await ButtonV1PageObject.waitForOnClickCallbackToFire(`The primary button failed to fire an onClick callback with a mouse click.`), @@ -74,7 +78,7 @@ describe('ButtonV1 Functional Testing', () => { .toBeFalsy(); }); - it('Type "Enter" on primary button. Validate onClick() callback was fired.', async () => { + desktopIt('Type "Enter" on primary button. Validate onClick() callback was fired.', { requires: ['keyboard'] }, async () => { await ButtonV1PageObject.sendKeys(ButtonV1PageObject._primaryComponent, [Keys.ENTER]); await expect( await ButtonV1PageObject.waitForOnClickCallbackToFire( @@ -87,7 +91,7 @@ describe('ButtonV1 Functional Testing', () => { .toBeFalsy(); }); - it('Type "Space" on primary button. Validate onClick() callback was fired.', async () => { + desktopIt('Type "Space" on primary button. Validate onClick() callback was fired.', { requires: ['keyboard'] }, async () => { await ButtonV1PageObject.sendKeys(ButtonV1PageObject._primaryComponent, [Keys.SPACE]); await expect( diff --git a/packages/agentic/desktop-driver/README.md b/packages/agentic/desktop-driver/README.md index e061eb895a..484c27638b 100644 --- a/packages/agentic/desktop-driver/README.md +++ b/packages/agentic/desktop-driver/README.md @@ -139,23 +139,24 @@ The [reference index](references/README.md) describes each package-owned guide. ## Public entry points -| Export | Purpose | -| --------------- | ----------------------------------------------------------------------------------------- | -| Package root | Native-helper build/resolution, `DesktopHost`, server, low-level client, and common types | -| `/authoring` | Serializable story plans, selectors, capabilities, and result types | -| `/wdio` | WebdriverIO connection, custom commands, assertions, and story-plan runner | -| `/client` | Low-level typed W3C client | -| `/authority` | Grants, private connection descriptors, and macOS owner-validated storage | -| `/native` | Native-helper build, resolution, and verification without loading the package root | -| `/server` | Embeddable remote end, session manager, and target registry | -| `/agent` | Bounded JSON-safe inspection, action, and evidence API | -| `/operations` | Versioned agent operation catalog, executor, and result types | -| `/artifacts` | Confined, atomic evidence persistence | -| `/runner` | Plan selection, sharding, execution, and result classification | -| `/storybook` | Story manifest and orchestration contracts, without a Storybook dependency | -| `/testing` | Fake host, fake Storybook orchestrator, and protocol harnesses | -| `/cli` | Embeddable Commander command and CLI runner | -| `/cli/reporter` | Shared reporter mode resolution and structured output framing | +| Export | Purpose | +| ------------------ | ----------------------------------------------------------------------------------------- | +| Package root | Native-helper build/resolution, `DesktopHost`, server, low-level client, and common types | +| `/authoring` | Serializable story plans, selectors, capabilities, result types, and case descriptors | +| `/wdio` | WebdriverIO connection, custom commands, assertions, and story-plan runner | +| `/client` | Low-level typed W3C client | +| `/authority` | Grants, private connection descriptors, and macOS owner-validated storage | +| `/native` | Native-helper build, resolution, and verification without loading the package root | +| `/server` | Embeddable remote end, session manager, and target registry | +| `/agent` | Bounded JSON-safe inspection, action, and evidence API | +| `/operations` | Versioned agent operation catalog, executor, and result types | +| `/artifacts` | Confined, atomic evidence persistence | +| `/runner` | Plan selection, sharding, execution, and result classification | +| `/storybook` | Story manifest and orchestration contracts, without a Storybook dependency | +| `/testing` | Fake host, fake Storybook orchestrator, and protocol harnesses | +| `/cli` | Embeddable Commander command and CLI runner | +| `/cli/reporter` | Shared reporter mode resolution and structured output framing | +| `/authoring/input` | Descriptor vocabulary and scope rules with no runtime imports (loads from source) | The service implements native application semantics rather than pretending to be a browser. Unsupported browser features return `unsupported operation` diff --git a/packages/agentic/desktop-driver/package.json b/packages/agentic/desktop-driver/package.json index 9764867493..bb7b57efae 100644 --- a/packages/agentic/desktop-driver/package.json +++ b/packages/agentic/desktop-driver/package.json @@ -53,6 +53,11 @@ "import": "./lib/authoring/index.js", "default": "./src/authoring/index.ts" }, + "./authoring/input": { + "types": "./lib/authoring/input.d.ts", + "import": "./lib/authoring/input.js", + "default": "./src/authoring/input.ts" + }, "./agent": { "types": "./lib/agent/index.d.ts", "import": "./lib/agent/index.js", diff --git a/packages/agentic/desktop-driver/references/architecture.md b/packages/agentic/desktop-driver/references/architecture.md index 13a181c733..ae89c8e124 100644 --- a/packages/agentic/desktop-driver/references/architecture.md +++ b/packages/agentic/desktop-driver/references/architecture.md @@ -120,25 +120,25 @@ handoff are still work in progress; see [current status](current-status.md). ## Module ownership -| Module | Responsibility | -| ------------------ | ------------------------------------------------------------------ | -| `src/protocol` | W3C parsing, actions, capabilities, timeouts, and errors | -| `src/server` | HTTP routing, target registry, sessions, windows, and elements | -| `src/host` | Platform-neutral `DesktopHost` and target contracts | -| `src/hosts/fake` | Deterministic in-memory provider | -| `src/hosts/native` | Long-lived helper process and `DesktopHost` adapter | -| `src/native` | Build, cache, resolution, verification, manifests, and wire types | -| `native/macos` | Swift operating-system implementation | -| `native/windows` | Shared C++ Windows and Win32 implementation | -| `src/client` | Low-level typed W3C client | -| `src/wdio` | Sanctioned WebdriverIO integration and custom commands | -| `src/authoring` | Serializable story plans, selectors, capabilities, and results | -| `src/runner` | Filtering, sharding, execution, and result classification | -| `src/artifacts` | Confined atomic evidence | -| `src/agent` | Bounded JSON-safe operations | -| `src/operations` | Agent operation catalog, executor, references, ledger, and journal | -| `src/testing` | Fake harnesses and Storybook test doubles | -| `src/cli` | Commander parsing, JSON output, and process exit adaptation | +| Module | Responsibility | +| ------------------ | -------------------------------------------------------------------------------- | +| `src/protocol` | W3C parsing, actions, capabilities, timeouts, and errors | +| `src/server` | HTTP routing, target registry, sessions, windows, and elements | +| `src/host` | Platform-neutral `DesktopHost` and target contracts | +| `src/hosts/fake` | Deterministic in-memory provider | +| `src/hosts/native` | Long-lived helper process and `DesktopHost` adapter | +| `src/native` | Build, cache, resolution, verification, manifests, and wire types | +| `native/macos` | Swift operating-system implementation | +| `native/windows` | Shared C++ Windows and Win32 implementation | +| `src/client` | Low-level typed W3C client | +| `src/wdio` | Sanctioned WebdriverIO integration and custom commands | +| `src/authoring` | Serializable story plans, selectors, capabilities, results, and case descriptors | +| `src/runner` | Filtering, sharding, execution, and result classification | +| `src/artifacts` | Confined atomic evidence | +| `src/agent` | Bounded JSON-safe operations | +| `src/operations` | Agent operation catalog, executor, references, ledger, and journal | +| `src/testing` | Fake harnesses and Storybook test doubles | +| `src/cli` | Commander parsing, JSON output, and process exit adaptation | ## Deliberate boundaries diff --git a/packages/agentic/desktop-driver/references/contributing.md b/packages/agentic/desktop-driver/references/contributing.md index 40e9c15a12..9d519cef3c 100644 --- a/packages/agentic/desktop-driver/references/contributing.md +++ b/packages/agentic/desktop-driver/references/contributing.md @@ -23,7 +23,7 @@ change-oriented guidance. | `src/native` | Source build, cache, selection, verification, and native artifact types | | `native/macos` | Swift implementation and self-test | | `native/windows` | Shared C++ Windows/Win32 implementation and self-test | -| `src/authoring` | Static JSON plan and result schemas | +| `src/authoring` | Static JSON plan and result schemas, and the case descriptor | | `src/runner` | Plan selection, capability checks, execution, and classification | | `src/wdio` | Sanctioned WebdriverIO connection and commands | | `src/client` | Low-level typed W3C client | @@ -38,6 +38,39 @@ Do not duplicate native build, cache, or verification logic in Storybook consumers. Do not move Storybook manifest generation or app lifecycle into this package. +## Case descriptors and input lanes + +`src/authoring/input.ts` and `src/authoring/selection.ts` are the one +implementation of the test case descriptor (`requires`, `input`, `platforms`, +`scope`, and declared gaps). Storybook extraction and its runner, and the +desktop-test Jasmine adapter, all call them; do not reimplement validation, +merging, or a decision in a consumer. Keep both files pure data and functions, +with no Node, React, or WebdriverIO imports, because stories and workers load +them. `input.ts` also has no runtime imports at all (its one import is a type), +and the `./authoring/input` subpath exports it directly: desktop-test's config +loader reads scope domains from it, and a config file is loaded from source +through a CommonJS fallback that cannot resolve this package's `.js` +specifiers, so keep it that way. + +- Add a requirement to `desktopRequirements`, to the physical or accessible + requirement set in `input.ts` when it implies an input, and to the + `requirementFeature` map in `selection.ts` for the host feature that backs it, + together. A requirement whose feature a host did not report is unmet. +- `mergeDesktopCaseOptions` only narrows: requirements union, `platforms` and + `scope` intersect, and a nested declaration can never widen or change its + group's input. An overlap that is empty is an error because the case could + never run. A group cannot carry a gap. +- `decideDesktopCase` judges platform, then scope, then the lane, so a case is + excluded for one reason. It resolves a declared gap too, so an expired + quarantine stops a run before anything executes. +- `resolveDesktopClickMode` lets a lane decide the click mode outright. That is + how one `click` requirement is an accessibility click in the accessible lane + and a physical click in the physical lane. +- `accessibleLaneMaskedFeatures` (`keyboard`, `physicalClick`, `wheel`) is what + an accessible lane disables through the native host's `disabledInputFeatures`, + and `unmaskedAccessibleFeatures` fails closed when a host reports any of them + as still available. Change the set together with the hosts that apply it. + ## Behavioral rules - Reserve a target before asynchronous launch or attach. diff --git a/packages/agentic/desktop-driver/references/test-integration.md b/packages/agentic/desktop-driver/references/test-integration.md index 82e57e60be..6e09fb1c46 100644 --- a/packages/agentic/desktop-driver/references/test-integration.md +++ b/packages/agentic/desktop-driver/references/test-integration.md @@ -98,15 +98,24 @@ exits, including on assertion failures, timeouts, or crashes. WebdriverIO's sets `waitforInterval` (100 ms) and `waitforTimeout` (5 s) explicitly; without an interval, `waitUntil` and `waitFor*` stop after their first unsatisfied poll. -Storybook's preferred top-level `wdio` functions and named function collections -use this pattern with Node's built-in test runner. Each function receives the -target `platform` (`macos`, `windows`, or `win32`), and each named case gets an -independent worker/session. Native feature flags are typed on the WebdriverIO -browser's `furn:features` capability. Button now uses this authoring pattern -instead of serializable plans. Extraction, native-bundle stripping, runner -configuration, and source freshness belong to -`@fluentui-react-native/storybook-desktop`, -not this Storybook-independent package. Static plans below are unchanged. +Storybook's preferred top-level `wdio` export is an array of `desktopIt` cases, +which declare the input and platforms they need. Its older functions and named +function collections use the same pattern with Node's built-in test runner. +Each function receives the target `platform` (`macos`, `windows`, or `win32`), +and each named case gets an independent worker/session. Native feature flags are +typed on the WebdriverIO browser's `furn:features` capability. Button now uses +this authoring pattern instead of serializable plans. Extraction, native-bundle +stripping, runner configuration, and source freshness belong to +`@fluentui-react-native/storybook-desktop`, not this Storybook-independent +package. Static plans below are unchanged. + +The descriptor that `desktopIt` accepts (`requires`, `input`, `platforms`, +`scope`, and declared gaps) is defined and decided in `./authoring`, so Storybook +and `desktop-test` share one implementation. A runner that executes one input lane +pins `furn:clickMode` with `resolveDesktopClickMode`: `element.click()` is then an +accessibility click in the accessible lane and a physical click in the physical +lane. `missingDesktopRequirements` judges a case's `requires` against the +session's `furn:features`, and judges a `click` requirement by the pinned mode. ## Author portable story plans diff --git a/packages/agentic/desktop-driver/src/authoring/index.ts b/packages/agentic/desktop-driver/src/authoring/index.ts index 0fd5cd6695..96cae618c8 100644 --- a/packages/agentic/desktop-driver/src/authoring/index.ts +++ b/packages/agentic/desktop-driver/src/authoring/index.ts @@ -14,15 +14,39 @@ export { desktopStoryPlatforms, validateDesktopStoryTests, } from './storyTests.js'; -export { desktopRequirements, matchesDesktopPlatforms, resolveDesktopInput, validateDesktopCaseOptions } from './input.js'; +export { + accessibleLaneMaskedFeatures, + desktopInputLanes, + desktopRequirements, + matchesDesktopPlatforms, + matchesDesktopScope, + mergeDesktopCaseOptions, + resolveDesktopInput, + resolveDesktopScopeDomains, + resolveDesktopScopeValues, + unmaskedAccessibleFeatures, + validateDesktopCaseOptions, + validateDesktopGroupOptions, +} from './input.js'; export type { + AccessibleLaneMaskedFeature, DesktopCaseOptions, DesktopInput, DesktopInputLane, DesktopPlatforms, DesktopRequirement, + DesktopScopeDomains, + DesktopScopeValues, ResolvedDesktopCaseOptions, } from './input.js'; +export { decideDesktopCase, missingDesktopRequirements, resolveDesktopClickMode } from './selection.js'; +export type { + DesktopCaseClickMode, + DesktopCaseDecision, + DesktopCaseTarget, + DesktopDeclaredGap, + DesktopExclusionReason, +} from './selection.js'; export type { DesktopStoryCapability, DesktopStoryExpectation, diff --git a/packages/agentic/desktop-driver/src/authoring/input.test.ts b/packages/agentic/desktop-driver/src/authoring/input.test.ts index 1c02abba19..0077d51cfc 100644 --- a/packages/agentic/desktop-driver/src/authoring/input.test.ts +++ b/packages/agentic/desktop-driver/src/authoring/input.test.ts @@ -1,4 +1,88 @@ -import { matchesDesktopPlatforms, resolveDesktopInput, validateDesktopCaseOptions } from './input.js'; +import { + accessibleLaneMaskedFeatures, + matchesDesktopPlatforms, + matchesDesktopScope, + mergeDesktopCaseOptions, + resolveDesktopInput, + resolveDesktopScopeDomains, + resolveDesktopScopeValues, + unmaskedAccessibleFeatures, + validateDesktopCaseOptions, + validateDesktopGroupOptions, +} from './input.js'; + +describe('accessible lane masks', () => { + test('lists the features a lane must hide, and fails closed when a host does not report them', () => { + expect(accessibleLaneMaskedFeatures).toEqual(['keyboard', 'physicalClick', 'wheel']); + expect(unmaskedAccessibleFeatures({ keyboard: false, physicalClick: false, wheel: false, focus: true })).toEqual([]); + expect(unmaskedAccessibleFeatures({ keyboard: true, physicalClick: false, wheel: false })).toEqual(['keyboard']); + expect(unmaskedAccessibleFeatures({ keyboard: false })).toEqual(['physicalClick', 'wheel']); + expect(unmaskedAccessibleFeatures(undefined)).toEqual(['keyboard', 'physicalClick', 'wheel']); + expect(unmaskedAccessibleFeatures('masked')).toEqual(['keyboard', 'physicalClick', 'wheel']); + }); +}); + +describe('desktop option groups', () => { + const group = (options: object) => validateDesktopGroupOptions(options); + const kase = (options: object) => validateDesktopCaseOptions(options); + + test('a group cannot declare a gap, because a gap describes one case', () => { + expect(() => group({ unsupported: { platforms: ['win32'], reason: 'No.' } })).toThrow('declare the gap on the case'); + expect(() => group({ quarantine: { issue: '#1', reason: 'Flaky.', expires: '2099-01-01' } })).toThrow('declare the gap on the case'); + }); + + test('composes requirements, platforms and scope, and derives the input from the union', () => { + const merged = mergeDesktopCaseOptions( + group({ requires: ['wheel'], platforms: ['!win32'], scope: { 'rn.renderer': ['fabric', 'paper'] } }), + kase({ requires: ['wheel', 'focus'], platforms: ['windows', 'win32'], scope: { 'rn.renderer': ['fabric'] } }), + ); + expect(merged).toEqual({ + requires: ['wheel', 'focus'], + platforms: ['windows'], + scope: { 'rn.renderer': ['fabric'] }, + input: 'physical', + inputSource: 'derived', + }); + }); + + test('returns the case unchanged outside any group', () => { + const options = kase({ requires: ['click'] }); + expect(mergeDesktopCaseOptions(undefined, options)).toBe(options); + }); + + test('lets a group narrow the input, but a nested declaration can never widen or switch it', () => { + const physical = group({ input: 'physical' }); + expect(mergeDesktopCaseOptions(physical, kase({}))).toMatchObject({ input: 'physical', inputSource: 'declared' }); + expect(mergeDesktopCaseOptions(physical, kase({ input: 'physical' }))).toMatchObject({ input: 'physical' }); + expect(() => mergeDesktopCaseOptions(physical, kase({ input: 'both' }))).toThrow('cannot change the enclosing group'); + expect(() => mergeDesktopCaseOptions(physical, kase({ input: 'accessible' }))).toThrow('cannot change the enclosing group'); + expect(() => mergeDesktopCaseOptions(physical, kase({ requires: ['accessibility-click'] }))).toThrow('cannot widen'); + }); + + test('keeps a group that requires a transport from being widened or mixed by a case', () => { + const keyboard = group({ requires: ['keyboard'] }); + expect(() => mergeDesktopCaseOptions(keyboard, kase({ input: 'both' }))).toThrow('cannot widen'); + expect(() => mergeDesktopCaseOptions(keyboard, kase({ requires: ['accessibility-click'] }))).toThrow('split the case'); + }); + + test('an explicit both on a group narrows nothing', () => { + const open = group({ input: 'both' }); + expect(mergeDesktopCaseOptions(open, kase({ requires: ['keyboard'] }))).toMatchObject({ input: 'physical', inputSource: 'derived' }); + expect(mergeDesktopCaseOptions(open, kase({}))).toMatchObject({ input: 'both', inputSource: 'derived' }); + }); + + test('rejects a case that could never run', () => { + expect(() => mergeDesktopCaseOptions(group({ platforms: ['win32'] }), kase({ platforms: ['!win32'] }))).toThrow('could never run'); + expect(() => + mergeDesktopCaseOptions(group({ scope: { 'rn.renderer': ['fabric'] } }), kase({ scope: { 'rn.renderer': ['paper'] } })), + ).toThrow('could never run'); + }); + + test('carries a case gap through its groups', () => { + const unsupported = { platforms: ['win32'], reason: 'Win32 keeps the cycle behavior.', issue: '#4377' } as const; + expect(mergeDesktopCaseOptions(group({ requires: ['keyboard'] }), kase({ unsupported }))).toMatchObject({ unsupported }); + }); +}); describe('desktop input descriptors', () => { test.each([ @@ -30,3 +114,42 @@ describe('desktop input descriptors', () => { expect(() => validateDesktopCaseOptions({ requires: ['input'] })).toThrow('unsupported requirement'); }); }); + +describe('desktop scope domains', () => { + const domains = resolveDesktopScopeDomains({ flavor: ['stable', 'insider'] }); + const supplied = resolveDesktopScopeValues({ 'rn.renderer': 'fabric', flavor: 'stable' }, domains); + + test('merges custom domains with the built-in renderer and reserves rn.*', () => { + expect(domains).toEqual({ 'rn.renderer': ['fabric', 'paper'], flavor: ['stable', 'insider'] }); + expect(resolveDesktopScopeDomains()).toEqual({ 'rn.renderer': ['fabric', 'paper'] }); + expect(() => resolveDesktopScopeDomains({ 'rn.renderer': ['x'] })).toThrow('reserved'); + expect(() => resolveDesktopScopeDomains({ 'rn.other': ['x'] })).toThrow('reserved'); + expect(() => resolveDesktopScopeDomains({ Flavor: ['x'] })).toThrow('invalid key'); + expect(() => resolveDesktopScopeDomains({ flavor: [] })).toThrow('non-empty'); + }); + + test('rejects supplied values outside their domain', () => { + expect(() => resolveDesktopScopeValues({ unknown: 'a' }, domains)).toThrow('unknown scope key'); + expect(() => resolveDesktopScopeValues({ 'rn.renderer': 'metal' }, domains)).toThrow('must be one of: fabric, paper'); + expect(() => resolveDesktopScopeValues({ flavor: 3 }, domains)).toThrow('must be one of'); + expect(() => resolveDesktopScopeValues({ constructor: 'a' }, domains)).toThrow('unknown scope key'); + }); + + test('ORs values within a key and ANDs keys', () => { + expect(matchesDesktopScope(undefined, supplied, domains)).toBe(true); + expect(matchesDesktopScope({ 'rn.renderer': ['fabric', 'paper'] }, supplied, domains)).toBe(true); + expect(matchesDesktopScope({ 'rn.renderer': ['paper'] }, supplied, domains)).toBe(false); + expect(matchesDesktopScope({ 'rn.renderer': ['fabric'], flavor: ['stable'] }, supplied, domains)).toBe(true); + expect(matchesDesktopScope({ 'rn.renderer': ['fabric'], flavor: ['insider'] }, supplied, domains)).toBe(false); + }); + + test('treats an undeclared key or value, or an unsupplied key, as a configuration error even when another key excludes', () => { + expect(() => matchesDesktopScope({ 'rn.renderer': ['paper'], missing: ['a'] }, supplied, domains)).toThrow( + 'unknown scope key "missing"', + ); + expect(() => matchesDesktopScope({ 'rn.renderer': ['paper'], flavor: ['beta'] }, supplied, domains)).toThrow('"beta"'); + expect(() => matchesDesktopScope({ flavor: ['stable'] }, resolveDesktopScopeValues({}, domains), domains)).toThrow('not supplied'); + const inherited: Record = JSON.parse('{"constructor":["a"]}'); + expect(() => matchesDesktopScope(inherited, supplied, domains)).toThrow('unknown scope key'); + }); +}); diff --git a/packages/agentic/desktop-driver/src/authoring/input.ts b/packages/agentic/desktop-driver/src/authoring/input.ts index ee0e353c51..b81575801e 100644 --- a/packages/agentic/desktop-driver/src/authoring/input.ts +++ b/packages/agentic/desktop-driver/src/authoring/input.ts @@ -16,6 +16,21 @@ export const desktopRequirements = [ export type DesktopRequirement = (typeof desktopRequirements)[number]; export type DesktopInput = 'accessible' | 'physical' | 'both'; export type DesktopInputLane = Exclude; +export const desktopInputLanes = ['accessible', 'physical'] as const satisfies readonly DesktopInputLane[]; + +/** + * The input an accessible lane keeps from reaching the application. A host that reports any of these as anything other + * than `false` cannot back the lane, because a case could then deliver physical input without anyone noticing. + */ +export const accessibleLaneMaskedFeatures = ['keyboard', 'physicalClick', 'wheel'] as const; +export type AccessibleLaneMaskedFeature = (typeof accessibleLaneMaskedFeatures)[number]; + +/** The masked features a host still offers. A missing report counts as unmasked, so the check fails closed. */ +export function unmaskedAccessibleFeatures(features: unknown): readonly AccessibleLaneMaskedFeature[] { + const reported = typeof features === 'object' && features !== null ? (features as Readonly>) : {}; + return accessibleLaneMaskedFeatures.filter((feature) => reported[feature] !== false); +} + export type DesktopPlatforms = | readonly [DesktopEndpoint, ...DesktopEndpoint[]] | readonly [`!${DesktopEndpoint}`, ...`!${DesktopEndpoint}`[]]; @@ -112,6 +127,78 @@ export function validateDesktopCaseOptions(value: unknown, source = 'desktop tes }; } +/** + * Validates the options of a group. A group scopes the cases inside it, so it can narrow where they run and add what they + * require, but a gap describes one case and cannot be declared on a group. + */ +export function validateDesktopGroupOptions(value: unknown, source = 'desktop group options'): ResolvedDesktopCaseOptions { + const options = validateDesktopCaseOptions(value, source); + if (options.unsupported || options.quarantine) { + throw new TypeError(`${source} cannot declare unsupported or quarantine; declare the gap on the case.`); + } + return options; +} + +/** + * Composes the options of an enclosing group with those of a nested group or case. Requirements add up, platforms and + * scope values intersect, and `input` can only narrow, so nothing inside a group can claim less than the group's own setup + * needs. A combination that could never run is an error rather than a silent exclusion. + */ +export function mergeDesktopCaseOptions( + parent: ResolvedDesktopCaseOptions | undefined, + child: ResolvedDesktopCaseOptions, + source = 'desktop test options', +): ResolvedDesktopCaseOptions { + if (!parent) return child; + const platforms = endpoints.filter( + (endpoint) => matchesDesktopPlatforms(parent.platforms, endpoint) && matchesDesktopPlatforms(child.platforms, endpoint), + ); + if (platforms.length === 0) throw new TypeError(`${source}.platforms do not overlap the enclosing group's, so it could never run.`); + const scope = intersectScope(parent.scope, child.scope, source); + const input = mergeDeclaredInput(parent, child, source); + return validateDesktopCaseOptions( + { + ...(input ? { input } : {}), + requires: [...new Set([...(parent.requires ?? []), ...(child.requires ?? [])])], + ...(platforms.length < endpoints.length ? { platforms } : {}), + ...(scope ? { scope } : {}), + ...(child.unsupported ? { unsupported: child.unsupported } : {}), + ...(child.quarantine ? { quarantine: child.quarantine } : {}), + }, + source, + ); +} + +function mergeDeclaredInput( + parent: ResolvedDesktopCaseOptions, + child: ResolvedDesktopCaseOptions, + source: string, +): DesktopInput | undefined { + // An explicit `both` narrows nothing, so it must not stop a nested case from deriving a transport. + const enclosing = parent.inputSource === 'declared' && parent.input !== 'both' ? parent.input : undefined; + if (child.inputSource !== 'declared') return enclosing; + if (enclosing && child.input !== enclosing) + throw new TypeError(`${source}.input "${child.input}" cannot change the enclosing group's "${enclosing}".`); + return child.input; +} + +function intersectScope( + parent: DesktopCaseOptions['scope'], + child: DesktopCaseOptions['scope'], + source: string, +): Record | undefined { + if (!parent && !child) return undefined; + const merged: Record = {}; + for (const key of new Set([...Object.keys(parent ?? {}), ...Object.keys(child ?? {})])) { + const left = parent?.[key]; + const right = child?.[key]; + const values = left && right ? left.filter((value) => right.includes(value)) : [...(left ?? right ?? [])]; + if (values.length === 0) throw new TypeError(`${source}.scope.${key} does not overlap the enclosing group's, so it could never run.`); + merged[key] = values; + } + return merged; +} + export function matchesDesktopPlatforms(platforms: DesktopPlatforms | undefined, endpoint: DesktopEndpoint): boolean { if (platforms === undefined) return true; const negative = platforms[0].startsWith('!'); @@ -119,6 +206,67 @@ export function matchesDesktopPlatforms(platforms: DesktopPlatforms | undefined, return negative ? !values.includes(endpoint) : values.includes(endpoint); } +/** Allowed values per scope key. */ +export type DesktopScopeDomains = Readonly>; +/** The one value a target holds for each scope key it supplies. */ +export type DesktopScopeValues = Readonly>; + +const builtInScopeDomains: DesktopScopeDomains = Object.freeze({ 'rn.renderer': Object.freeze(['fabric', 'paper']) }); + +/** + * The built-in domains plus any custom ones. A consumer adds its own keys, such as a product flavor, but `rn.*` belongs to + * the framework so a test written for `rn.renderer` means the same thing everywhere. + */ +export function resolveDesktopScopeDomains(custom?: unknown, source = 'testScopes'): DesktopScopeDomains { + if (custom === undefined) return builtInScopeDomains; + const resolved: Record = { ...builtInScopeDomains }; + for (const [key, values] of Object.entries(object(custom, source))) { + if (key.startsWith('rn.')) throw new TypeError(`${source} cannot declare "${key}"; the rn.* scope keys are reserved.`); + resolved[key] = Object.freeze([...scopeEntry(key, values, source)]); + } + return Object.freeze(resolved); +} + +/** Checks that every key a target supplies is a known domain and that its value belongs to it. */ +export function resolveDesktopScopeValues(supplied: unknown, domains: DesktopScopeDomains, source = 'testScope'): DesktopScopeValues { + const resolved: Record = {}; + for (const [key, value] of Object.entries(object(supplied, source))) { + const domain = ownDomain(domains, key); + if (!domain) throw new TypeError(`${source} uses unknown scope key "${key}".`); + if (typeof value !== 'string' || !domain.includes(value)) throw new TypeError(`${source}.${key} must be one of: ${domain.join(', ')}.`); + resolved[key] = value; + } + return Object.freeze(resolved); +} + +/** + * Values OR within a key and keys AND. An undeclared key or value, or a key the target does not supply, is a + * configuration error rather than an exclusion, so a typo cannot silently drop a case. Every key is checked before the + * result is returned. + */ +export function matchesDesktopScope( + scope: DesktopCaseOptions['scope'], + supplied: DesktopScopeValues, + domains: DesktopScopeDomains, + source = 'desktop test scope', +): boolean { + let matches = true; + for (const [key, values] of Object.entries(scope ?? {})) { + const domain = ownDomain(domains, key); + if (!domain) throw new TypeError(`${source} uses unknown scope key "${key}".`); + const unknown = values.find((value) => !domain.includes(value)); + if (unknown !== undefined) throw new TypeError(`${source}.${key} uses "${unknown}", which is not one of: ${domain.join(', ')}.`); + const actual = Object.hasOwn(supplied, key) ? supplied[key] : undefined; + if (actual === undefined) throw new TypeError(`${source}.${key} is not supplied by this target.`); + if (!values.includes(actual)) matches = false; + } + return matches; +} + +function ownDomain(domains: DesktopScopeDomains, key: string): readonly string[] | undefined { + return Object.hasOwn(domains, key) ? domains[key] : undefined; +} + function validatePlatforms(value: unknown, source: string, required = false): void { if (value === undefined && !required) return; if (!Array.isArray(value) || value.length === 0) throw new TypeError(`${source} must be a non-empty platform list.`); @@ -138,19 +286,21 @@ function validatePlatforms(value: unknown, source: string, required = false): vo function validateScope(value: unknown, source: string): void { if (value === undefined) return; - const scope = object(value, source); - for (const [key, values] of Object.entries(scope)) { - if (!/^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)*$/.test(key)) throw new TypeError(`${source} contains invalid key "${key}".`); - if (!Array.isArray(values) || values.length === 0) throw new TypeError(`${source}.${key} must be a non-empty value list.`); - const seen = new Set(); - for (const item of values) { - if (typeof item !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(item)) { - throw new TypeError(`${source}.${key} contains an invalid value.`); - } - if (seen.has(item)) throw new TypeError(`${source}.${key} contains duplicate "${item}".`); - seen.add(item); + for (const [key, values] of Object.entries(object(value, source))) scopeEntry(key, values, source); +} + +function scopeEntry(key: string, values: unknown, source: string): readonly string[] { + if (!/^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)*$/.test(key)) throw new TypeError(`${source} contains invalid key "${key}".`); + if (!Array.isArray(values) || values.length === 0) throw new TypeError(`${source}.${key} must be a non-empty value list.`); + const seen = new Set(); + for (const item of values) { + if (typeof item !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(item)) { + throw new TypeError(`${source}.${key} contains an invalid value.`); } + if (seen.has(item)) throw new TypeError(`${source}.${key} contains duplicate "${item}".`); + seen.add(item); } + return values as readonly string[]; } function object(value: unknown, source: string): Record { diff --git a/packages/agentic/desktop-driver/src/authoring/selection.test.ts b/packages/agentic/desktop-driver/src/authoring/selection.test.ts new file mode 100644 index 0000000000..2a8505dafc --- /dev/null +++ b/packages/agentic/desktop-driver/src/authoring/selection.test.ts @@ -0,0 +1,96 @@ +import { resolveDesktopScopeDomains, resolveDesktopScopeValues, validateDesktopCaseOptions } from './input.js'; +import { decideDesktopCase, missingDesktopRequirements, resolveDesktopClickMode } from './selection.js'; +import type { DesktopCaseTarget } from './selection.js'; + +describe('decideDesktopCase', () => { + const domains = resolveDesktopScopeDomains(); + const target = (overrides: Partial = {}): DesktopCaseTarget => ({ + endpoint: 'windows', + domains, + values: resolveDesktopScopeValues({ 'rn.renderer': 'fabric' }, domains), + now: Date.parse('2026-01-01T00:00:00Z'), + ...overrides, + }); + const decide = (options: object, overrides?: Partial) => + decideDesktopCase(validateDesktopCaseOptions(options), target(overrides), 'story / case'); + + test('runs a case that applies, and ignores the input outside a lane', () => { + expect(decide({})).toEqual({ kind: 'run' }); + expect(decide({ requires: ['keyboard'] })).toEqual({ kind: 'run' }); + expect(decide({ requires: ['keyboard'] }, { lane: 'physical' })).toEqual({ kind: 'run' }); + }); + + test('excludes by the first applicable reason: platform, then scope, then input', () => { + const options = { platforms: ['macos'], scope: { 'rn.renderer': ['paper'] }, requires: ['keyboard'] }; + expect(decide(options, { lane: 'accessible' })).toEqual({ kind: 'excluded', reason: 'platform' }); + expect(decide({ ...options, platforms: ['windows'] }, { lane: 'accessible' })).toEqual({ kind: 'excluded', reason: 'scope' }); + expect(decide({ ...options, platforms: ['windows'], scope: { 'rn.renderer': ['fabric'] } }, { lane: 'accessible' })).toEqual({ + kind: 'excluded', + reason: 'input', + }); + }); + + test('keeps a case that runs in both lanes', () => { + expect(decide({ requires: ['click'] }, { lane: 'accessible' })).toEqual({ kind: 'run' }); + expect(decide({ requires: ['click'] }, { lane: 'physical' })).toEqual({ kind: 'run' }); + }); + + test('resolves a declared unsupported case as a gap instead of running or excluding it', () => { + const unsupported = { platforms: ['windows'], reason: 'Not projected yet.' }; + expect(decide({ unsupported })).toEqual({ kind: 'gap', gap: { kind: 'unsupported', reason: 'Not projected yet.' } }); + expect(decide({ unsupported }, { endpoint: 'macos' })).toEqual({ kind: 'run' }); + }); + + test('applies a quarantine to its lane and platforms, and rejects an expired one before anything runs', () => { + const quarantine = { issue: '#9', reason: 'Flaky.', expires: '2026-06-01', input: 'physical', platforms: ['windows'] }; + expect(decide({ quarantine }, { lane: 'physical' })).toEqual({ kind: 'gap', gap: { kind: 'quarantine', reason: '#9: Flaky.' } }); + expect(decide({ quarantine }, { lane: 'accessible' })).toEqual({ kind: 'run' }); + expect(decide({ quarantine })).toEqual({ kind: 'run' }); + expect(decide({ quarantine }, { endpoint: 'win32', lane: 'physical' })).toEqual({ kind: 'run' }); + expect(() => decide({ quarantine }, { lane: 'physical', now: Date.parse('2026-06-02T00:00:00Z') })).toThrow( + 'Quarantine #9 expired on 2026-06-01.', + ); + }); +}); + +describe('click mode and requirement preflight', () => { + test('adapts a click requirement to the lane without falling back to the other transport', () => { + const descriptor = validateDesktopCaseOptions({ requires: ['click'] }); + const accessibleMode = resolveDesktopClickMode(descriptor, 'auto', 'accessible'); + const physicalMode = resolveDesktopClickMode(descriptor, 'auto', 'physical'); + expect(accessibleMode).toBe('accessibility'); + expect(physicalMode).toBe('physical'); + expect(missingDesktopRequirements(descriptor, { accessibilityClick: true, physicalClick: false }, physicalMode)).toEqual(['click']); + expect(missingDesktopRequirements(descriptor, { accessibilityClick: false, physicalClick: true }, accessibleMode)).toEqual(['click']); + expect(missingDesktopRequirements(descriptor, { accessibilityClick: true }, accessibleMode)).toEqual([]); + }); + + test('outside a lane only an accessible case pins its click transport, and the rest keep the configured mode', () => { + expect(resolveDesktopClickMode(validateDesktopCaseOptions({ requires: ['accessibility-click'] }), 'auto', undefined)).toBe( + 'accessibility', + ); + expect(resolveDesktopClickMode(validateDesktopCaseOptions({ input: 'accessible' }), 'physical', undefined)).toBe('accessibility'); + // A physical case must not pin physical click, or a host that masks it fails the session instead of reporting unsupported. + expect(resolveDesktopClickMode(validateDesktopCaseOptions({ requires: ['physical-click'] }), 'auto', undefined)).toBe('auto'); + expect(resolveDesktopClickMode(validateDesktopCaseOptions({ requires: ['keyboard'] }), 'auto', undefined)).toBe('auto'); + expect(resolveDesktopClickMode(validateDesktopCaseOptions({ requires: ['click'] }), 'physical', undefined)).toBe('physical'); + expect(resolveDesktopClickMode(undefined, 'auto', undefined)).toBe('auto'); + }); + + test('an automatic click needs either transport, and an absent report meets nothing', () => { + const click = validateDesktopCaseOptions({ requires: ['click'] }); + expect(missingDesktopRequirements(click, { physicalClick: false, accessibilityClick: false }, 'auto')).toEqual(['click']); + expect(missingDesktopRequirements(click, { physicalClick: true }, 'auto')).toEqual([]); + expect(missingDesktopRequirements(validateDesktopCaseOptions({ requires: ['keyboard', 'focus'] }), undefined, 'auto')).toEqual([ + 'keyboard', + 'focus', + ]); + expect( + missingDesktopRequirements( + validateDesktopCaseOptions({ requires: ['element-screenshot', 'set-window-rect'] }), + { elementScreenshot: true, setWindowRect: false }, + 'auto', + ), + ).toEqual(['set-window-rect']); + }); +}); diff --git a/packages/agentic/desktop-driver/src/authoring/selection.ts b/packages/agentic/desktop-driver/src/authoring/selection.ts new file mode 100644 index 0000000000..383ce5896f --- /dev/null +++ b/packages/agentic/desktop-driver/src/authoring/selection.ts @@ -0,0 +1,116 @@ +import type { DesktopEndpoint } from '../protocol/types.js'; +import { matchesDesktopPlatforms, matchesDesktopScope } from './input.js'; +import type { DesktopInputLane, DesktopRequirement, DesktopScopeDomains, DesktopScopeValues, ResolvedDesktopCaseOptions } from './input.js'; + +/** Why a declared case is not part of a run. Excluded cases are not tests, so they are counted and never reported as skipped. */ +export type DesktopExclusionReason = 'platform' | 'scope' | 'input'; + +/** A gap the author declared in advance. The case is selected and reported as unsupported, not executed. */ +export type DesktopDeclaredGap = { kind: 'unsupported' | 'quarantine'; reason: string }; + +/** What a case is judged against: the endpoint and scope values of the target, and the input lane of the run if it has one. */ +export type DesktopCaseTarget = { + endpoint: DesktopEndpoint; + lane?: DesktopInputLane; + domains: DesktopScopeDomains; + values: DesktopScopeValues; + /** Clock for quarantine expiry. */ + now?: number; +}; + +export type DesktopCaseDecision = + | { kind: 'excluded'; reason: DesktopExclusionReason } + | { kind: 'gap'; gap: DesktopDeclaredGap } + | { kind: 'run' }; + +/** + * Decides one declared case before it executes. The first applicable reason wins, in platform, scope, input order, so a + * case never counts against two exclusions. A declared gap is resolved here too, which is why an expired quarantine + * stops the run before anything executes. + */ +export function decideDesktopCase(descriptor: ResolvedDesktopCaseOptions, target: DesktopCaseTarget, source: string): DesktopCaseDecision { + if (!matchesDesktopPlatforms(descriptor.platforms, target.endpoint)) return { kind: 'excluded', reason: 'platform' }; + if (!matchesDesktopScope(descriptor.scope, target.values, target.domains, `${source} scope`)) + return { kind: 'excluded', reason: 'scope' }; + if (target.lane && descriptor.input !== 'both' && descriptor.input !== target.lane) return { kind: 'excluded', reason: 'input' }; + const gap = declaredGap(descriptor, target); + return gap ? { kind: 'gap', gap } : { kind: 'run' }; +} + +/** How a session turns a click into input. `auto` lets the host pick; the other two pin one transport. */ +export type DesktopCaseClickMode = 'auto' | 'physical' | 'accessibility'; + +/** + * The click mode a case runs under. A lane decides it outright, so a `click` requirement is an accessible click in the + * accessible lane and a physical click in the physical lane. Without a lane only an accessible case pins its transport, + * so it never moves the pointer. Any other case keeps the configured mode: a physical case on a host that masks physical + * click is then reported unsupported by the requirement preflight, as the skip it replaces was, instead of failing to + * create its session. + */ +export function resolveDesktopClickMode( + descriptor: ResolvedDesktopCaseOptions | undefined, + configured: DesktopCaseClickMode, + lane: DesktopInputLane | undefined, +): DesktopCaseClickMode { + if (lane === 'accessible') return 'accessibility'; + if (lane === 'physical') return 'physical'; + if (descriptor?.input === 'accessible' || descriptor?.requires?.includes('accessibility-click')) return 'accessibility'; + return configured; +} + +/** The requirements a host cannot meet. `features` is what the host reported for the session; an absent report meets nothing. */ +export function missingDesktopRequirements( + descriptor: ResolvedDesktopCaseOptions | undefined, + features: Readonly> | undefined, + clickMode: DesktopCaseClickMode, +): DesktopRequirement[] { + if (!descriptor?.requires?.length) return []; + const missing: DesktopRequirement[] = []; + for (const requirement of descriptor.requires) { + if (requirement === 'click') { + const met = + clickMode === 'physical' + ? features?.physicalClick === true + : clickMode === 'accessibility' + ? features?.accessibilityClick === true + : features?.physicalClick === true || features?.accessibilityClick === true; + if (!met) missing.push(requirement); + } else if (features?.[requirementFeature[requirement]] !== true) { + missing.push(requirement); + } + } + return missing; +} + +/** The host feature that backs each requirement other than `click`, which depends on the click mode. */ +const requirementFeature: Record, string> = { + 'accessibility-actions': 'accessibilityActions', + 'accessibility-click': 'accessibilityClick', + 'element-screenshot': 'elementScreenshot', + focus: 'focus', + keyboard: 'keyboard', + 'physical-click': 'physicalClick', + screenshot: 'screenshot', + 'set-window-rect': 'setWindowRect', + wheel: 'wheel', +}; + +function declaredGap( + descriptor: ResolvedDesktopCaseOptions, + { endpoint, lane, now = Date.now() }: DesktopCaseTarget, +): DesktopDeclaredGap | undefined { + if (descriptor.unsupported && matchesDesktopPlatforms(descriptor.unsupported.platforms, endpoint)) { + return { kind: 'unsupported', reason: descriptor.unsupported.reason }; + } + const quarantine = descriptor.quarantine; + if ( + quarantine && + (!quarantine.platforms || matchesDesktopPlatforms(quarantine.platforms, endpoint)) && + (!quarantine.input || quarantine.input === lane) + ) { + if (Date.parse(`${quarantine.expires}T00:00:00Z`) < now) + throw new Error(`Quarantine ${quarantine.issue} expired on ${quarantine.expires}.`); + return { kind: 'quarantine', reason: `${quarantine.issue}: ${quarantine.reason}` }; + } + return undefined; +} diff --git a/packages/agentic/desktop-driver/src/index.ts b/packages/agentic/desktop-driver/src/index.ts index 3e556374bc..0479787d78 100644 --- a/packages/agentic/desktop-driver/src/index.ts +++ b/packages/agentic/desktop-driver/src/index.ts @@ -32,15 +32,39 @@ export { desktopStoryPlatforms, validateDesktopStoryTests, } from './authoring/storyTests.js'; -export { desktopRequirements, matchesDesktopPlatforms, resolveDesktopInput, validateDesktopCaseOptions } from './authoring/input.js'; +export { + accessibleLaneMaskedFeatures, + desktopInputLanes, + desktopRequirements, + matchesDesktopPlatforms, + matchesDesktopScope, + mergeDesktopCaseOptions, + resolveDesktopInput, + resolveDesktopScopeDomains, + resolveDesktopScopeValues, + unmaskedAccessibleFeatures, + validateDesktopCaseOptions, + validateDesktopGroupOptions, +} from './authoring/input.js'; export type { + AccessibleLaneMaskedFeature, DesktopCaseOptions, DesktopInput, DesktopInputLane, DesktopPlatforms, DesktopRequirement, + DesktopScopeDomains, + DesktopScopeValues, ResolvedDesktopCaseOptions, } from './authoring/input.js'; +export { decideDesktopCase, missingDesktopRequirements, resolveDesktopClickMode } from './authoring/selection.js'; +export type { + DesktopCaseClickMode, + DesktopCaseDecision, + DesktopCaseTarget, + DesktopDeclaredGap, + DesktopExclusionReason, +} from './authoring/selection.js'; export type { DesktopStoryCapability, DesktopStoryExpectation, diff --git a/packages/agentic/desktop-test/package.json b/packages/agentic/desktop-test/package.json index f86c5cecd6..8a01e622bb 100644 --- a/packages/agentic/desktop-test/package.json +++ b/packages/agentic/desktop-test/package.json @@ -55,6 +55,11 @@ "import": "./lib/driverService.js", "default": "./src/driverService.ts" }, + "./jasmine": { + "types": "./lib/jasmine/index.d.ts", + "import": "./lib/jasmine/index.js", + "default": "./src/jasmine/index.ts" + }, "./wdio": { "types": "./lib/wdio/index.d.ts", "import": "./lib/wdio/index.js", diff --git a/packages/agentic/desktop-test/references/commands.md b/packages/agentic/desktop-test/references/commands.md index 47666999c4..817553b4bc 100644 --- a/packages/agentic/desktop-test/references/commands.md +++ b/packages/agentic/desktop-test/references/commands.md @@ -10,14 +10,15 @@ owner_paths: Expose `desktop-test` from the consumer workspace and run through that workspace's declared script. -| Command | Purpose | -| -------------------------------- | ---------------------------------------------------- | -| `build-driver` | Explicitly build or verify the native helper | -| `prep`, `bundle`, `build`, `run` | Run the selected target adapter stage | -| `test --suite --list` | Resolve groups without launching a target | -| `test --suite ` | Run the selected WDIO suite | -| `serve --reporter json` | Start the real driver listener and required services | -| `serve --agent` | Also require grants and publish an agent descriptor | +| Command | Purpose | +| ------------------------------------ | ---------------------------------------------------- | +| `build-driver` | Explicitly build or verify the native helper | +| `prep`, `bundle`, `build`, `run` | Run the selected target adapter stage | +| `test --suite --list` | Resolve groups without launching a target | +| `test --suite ` | Run the selected WDIO suite | +| `test --suite --input ` | Run one input lane (`accessible` or `physical`) | +| `serve --reporter json` | Start the real driver listener and required services | +| `serve --agent` | Also require grants and publish an agent descriptor | Global reporter options are `--reporter auto|plain|github-actions|json|ndjson` and `--json` (an alias for `--reporter json`). Mode precedence is CLI flag, @@ -58,6 +59,24 @@ For macOS RN Test App projects, run bundle before `prep`; CocoaPods embeds only resources present during pod installation. Changing the renderer requires rerunning `prep`. See [platform setup](platforms.md). +## Input lanes + +`test --input accessible|physical` runs one input lane. `DESKTOP_INPUT` supplies +the lane when the flag is absent, and an empty or unknown value is an error. The +`accessible` lane starts the native host with keyboard, physical click, and wheel +disabled and runs the tests that need no physical input; the `physical` lane +runs tests that need it plus those that need neither. Tests declare which with +the [`./jasmine` helpers](wdio-suites.md#declare-input-and-scope-in-jasmine-specs). +A lane requires a Jasmine WDIO suite (or the Storybook suite, whose integration +applies the lane itself) and rejects `--list`, which resolves spec files without +regard to a lane. Without a lane every declared test runs, filtered by `platforms` +and `scope`. See [input lanes in reporting](reporting.md#input-lanes) for how a +lane run is judged. + +An imported integration that serves its own driver receives the lane and owns +masking that driver. A host supplied through `hostFactory` is not masked by the +harness, so the suite verifies the mask before running anything. + The Storybook `csf-wdio` suite does not support generic `--list`. Discover story IDs with the Storybook integration's manifest/CLI, then select authored tests with its test filter. Use `storybook-desktop` smoke commands for a complete diff --git a/packages/agentic/desktop-test/references/configuration.md b/packages/agentic/desktop-test/references/configuration.md index 7895d23d4c..4a780442a7 100644 --- a/packages/agentic/desktop-test/references/configuration.md +++ b/packages/agentic/desktop-test/references/configuration.md @@ -37,6 +37,34 @@ The config schema rejects unknown fields, cycles, and unmatched spec patterns. Spec paths resolve against the suite project's root; target references and hooks resolve against their declaring config. +## Scope domains + +A test can declare `scope` values so it runs only on some targets, such as one +renderer or product flavor. Domains are declared in config. `rn.renderer` +(`fabric`, `paper`) is built in and comes from the target's React Native host; +the `rn.*` keys are reserved for the framework. A consumer that is not React +Native, or that needs another dimension, lists its own keys in `testScopes`, each +with a non-empty list of allowed values, and each target sets its own value in +`testScope`: + +```ts +export default defineDesktopConfig({ + schemaVersion: 1, + projectRoot: new URL('.', import.meta.url), + testScopes: { flavor: ['stable', 'preview'] }, + targets: { + windows: { + host: { kind: 'installed', application: { executablePath: 'C:\\apps\\Sample.exe', windowTitle: 'Sample' } }, + testScope: { flavor: 'preview' }, + }, + }, +}); +``` + +A `testScope` key that is not declared, a value outside its domain, or an attempt +to set an `rn.*` key is a configuration error. So is a test scoped on a key its +target does not supply: a typo or a missing value must not silently drop tests. + ## Helper policy `nativeDriver` accepts only `cacheRoot`, `helperPath`, and `installRoot`. diff --git a/packages/agentic/desktop-test/references/contributing.md b/packages/agentic/desktop-test/references/contributing.md index d5f7175b95..ee9a698cc6 100644 --- a/packages/agentic/desktop-test/references/contributing.md +++ b/packages/agentic/desktop-test/references/contributing.md @@ -12,7 +12,18 @@ CLI stages are composed in `src/cli.ts`; process ownership is in `src/supervisor.ts`; shared native service, recovery, authority, descriptor and settled shutdown composition lives in the real `src/driverService.ts` module (also used by the Storybook channel child); reports are implemented in `src/reporting/`; and WDIO -session wiring is in `src/wdio/`. +session wiring is in `src/wdio/`. The `./jasmine` entry (`src/jasmine/`) holds the +declaration helpers and the selection adapter that WDIO Jasmine suites use; input +lane accounting is in `src/reporting/lane.ts`. Descriptor validation and per-case +decisions come from `@fluentui-react-native/desktop-driver/authoring`, not from +copies here. + +`src/config.ts` is also loaded from source through the CommonJS prebuild fallback +that Storybook uses for `desktop.config.mts`, which cannot resolve desktop-driver's +`.js` specifiers. Its runtime imports are therefore Node built-ins, sibling `.ts` +modules and the leaf `@fluentui-react-native/desktop-driver/authoring/input`; import +everything else from desktop-driver as types only. The storybook-desktop +`sourceConfig` test guards this. Add contract coverage for schema, lifecycle, reporting protocol, and failure paths. Do not move Storybook-specific story extraction into this package. diff --git a/packages/agentic/desktop-test/references/reporting.md b/packages/agentic/desktop-test/references/reporting.md index 164b05dc64..4e91dc7319 100644 --- a/packages/agentic/desktop-test/references/reporting.md +++ b/packages/agentic/desktop-test/references/reporting.md @@ -65,6 +65,58 @@ directory or render a duplicate summary. Story-specific readiness, authenticated channel events, and callback execution are owned by `@fluentui-react-native/storybook-desktop`. +## Input lanes + +A `test` run can execute one input lane (see [commands](commands.md#input-lanes)). +Its report records the lane, counts what the run left out before executing, and +holds every selected test to its declared outcome. `run.json` carries the lane as +`{ input, source, masking?, policyEnforced }`: + +- `input`: `accessible` or `physical`. +- `source`: `option` for `--input`, `environment` for `DESKTOP_INPUT`, or `driver` + when the lane was read from the driver instance a `test` attached to. +- `masking` (accessible only): `host-masked` when this command started the host + with keyboard, physical click, and wheel disabled, or `verified-masked` when it + attached to a host and verified that mask before running anything. +- `policyEnforced`: always `false`. The report records what was requested and how + the host was masked; it never claims what input a test delivered, so a physical + lane is labelled a pilot. + +Tests that a lane, a `platforms` list, or a `scope` leaves out are not tests, so +they are never reported as skipped. A producer announces them in `selection` +events, and `run.json` sums them as `selection.excluded: { platform, scope, input }`. +A case counts once, under the first reason that applies in that order, and only if +it matched the user's own filters, such as a story filter. A `selection` event +with a `key` is counted once, so a retried worker that announces the same +exclusions again does not double them. A run whose producers reported none has no +`selection`. + +A lane run promised to execute its whole selection, so its verdict is stricter: + +- `failed`: a failed test or an infrastructure failure, as in any run. +- `incomplete`: a selected test was skipped at run time, was `unsupported` with no + declaration (the host did not meet a `requires`), or no test executed. `optional` + never excuses these, so a lane that runs nothing is not reported `skipped`. +- `passed`: otherwise. A test declared `unsupported` or `quarantine` is reported + unsupported without making the run incomplete. Summaries count them as + `N unsupported (N declared, N quarantined)` so an undeclared gap stands out. + +`failed` outranks `incomplete`, which outranks `passed`. A run without a lane keeps +its existing verdict. + +The plain and GitHub summaries show the lane and exclusions on one line, such as +`Selection: input accessible (host-masked) from --input; excluded 4 by platform, 0 by scope, 12 by input`. +The JUnit file records the lane as the properties `desktop.input`, +`desktop.input.label`, and `desktop.input.source`, the exclusions as +`desktop.excluded.platform`, `desktop.excluded.scope`, and `desktop.excluded.input`, +and `desktop.declaredUnsupported` and `desktop.quarantined` when present. The label +is `accessible (host-masked)`, `accessible (verified-masked)`, or `physical pilot`. + +A delegated command, which inherits its parent's collector, sends its exclusions to +the parent but not its lane, so the parent's `run.json` has no `lane`. The command +still holds its own cohort to the lane: a Storybook `test` reports an incomplete +cohort to the parent when a selected test has no declared gap and did not run. + ## Persisted read queries The direct `@fluentui-react-native/desktop-test/reporting/run-queries` entry @@ -117,7 +169,8 @@ external integration); `addSecret` can update the read redactor. The collector redacts before persistence, and queries redact again before clipping text. Readers submit only an opaque run ID, a declared operation, and its bounded -parameters. `status` returns complete counts and terminal/coverage state. +parameters. `status` returns complete counts, terminal/coverage state, and the +run's input `lane` and `excluded` counts when it has them. `tests`, `work`, `failures`, and `artifacts` return pages with a default limit of 10 and maximum of 25, capped at 16 KiB. Page IDs do not expose producer IDs or paths. Text fields are capped at 1 KiB after redaction. `omissions.items` counts diff --git a/packages/agentic/desktop-test/references/wdio-suites.md b/packages/agentic/desktop-test/references/wdio-suites.md index bf6a716e74..fa80879ec3 100644 --- a/packages/agentic/desktop-test/references/wdio-suites.md +++ b/packages/agentic/desktop-test/references/wdio-suites.md @@ -40,3 +40,68 @@ Prefer unique ids. A suite that depends on first-match lookup can set Component story callback extraction is owned by `@fluentui-react-native/storybook-desktop`; use this suite guide for ordinary WDIO suites and app-level E2E adapters. + +## Declare input and scope in Jasmine specs + +Declare each test with `desktopIt` and group tests with `desktopDescribe` from +`@fluentui-react-native/desktop-test/jasmine`. They take the same descriptor as +Storybook's `desktopIt` (`requires`, `input`, `platforms`, `scope`, `unsupported`, +`quarantine`), and the options argument is optional: + +```ts +import { desktopDescribe, desktopIt } from '@fluentui-react-native/desktop-test/jasmine'; + +desktopDescribe('Button', () => { + desktopIt('renders', async () => { + await expect(await $('~button')).toBeDisplayed(); + }); + desktopIt('activates with Enter', { requires: ['keyboard'] }, async () => { + await browser.keys('\uE007'); + }); + desktopDescribe('Windows behavior', { platforms: ['windows', 'win32'] }, () => { + desktopIt('takes focus', { requires: ['focus'] }, async () => { + await expect(await $('~button')).toBeFocused(); + }); + }); +}); +``` + +A group's options apply to everything inside it. Requirements add up, `platforms` +and `scope` intersect, and a nested declaration can only narrow its group, so a +combination that could never run is an error rather than a silent exclusion. A +group cannot declare `unsupported` or `quarantine`; declare the gap on the test. +A test's `input` comes from its `requires` unless declared; see "Declare how a case runs" in +`@fluentui-react-native/storybook-desktop/references/story-tests.md`. +A test that needs no input stays `both`. + +Run through `desktop-test test --suite `, the harness hands each WDIO worker +its endpoint, scope values, and optional lane as data in `DESKTOP_TEST_SELECTION`. +Once the session exists and before any test runs, the adapter filters Jasmine's +`specFilter`, so an excluded test never runs its hooks or body. Excluded tests +are counted in the run's selection totals, not reported as skipped. A declared +`unsupported` or active `quarantine` test is reported unsupported with its reason, +and an expired quarantine fails that test. A test whose `requires` the host cannot +meet is excluded the same way and reported unsupported. + +- **Lanes need declared tests.** An input lane runs only tests declared with + `desktopIt`. A test registered with plain `it` fails the run, which lists the first + ten, so migrate a whole file before running a lane against it. A plain `describe` + is fine; use `desktopDescribe` to scope its tests. Without a lane, plain specs run + unchanged. +- **Click mode is session-wide.** WDIO owns the session, so the lane sets + `furn:clickMode` once: accessibility click in the accessible lane, physical + click in the physical lane. A `click` requirement follows it. Without a lane the + session uses `auto`, which lets the host choose. +- **Untrustworthy selections fail closed.** A malformed selection, an unverifiable + Jasmine environment, or an accessible lane on a host that does not report keyboard, + physical click, and wheel disabled disables every test and fails the run. +- **A bad declaration never throws.** Jasmine only warns when a spec file fails to + load, so a throw would silently drop every test after it. An invalid declaration + registers a failing test that reports the problem. +- **Run outside the harness.** With no selection, such as plain `wdio`, every test + runs and descriptors are still validated. +- `desktopIt` takes no timeout argument; the suite's `timeoutMs` applies. + +The input-call lint of Storybook cases does not apply here. Page objects and helpers +hide the input a test sends, so the accessible lane's host mask is the safeguard that +fails a test that reaches for physical input. diff --git a/packages/agentic/desktop-test/src/cli.ts b/packages/agentic/desktop-test/src/cli.ts index 833b7b1c29..3258f8a45e 100644 --- a/packages/agentic/desktop-test/src/cli.ts +++ b/packages/agentic/desktop-test/src/cli.ts @@ -11,15 +11,17 @@ import { tolerateClosedDesktopStdout, } from '@fluentui-react-native/desktop-driver/cli/reporter'; import type { ResolvedDesktopReporterMode } from '@fluentui-react-native/desktop-driver/cli/reporter'; +import { desktopInputLanes } from '@fluentui-react-native/desktop-driver/authoring'; import { desktopStageCommands } from './adapters.js'; import { defaultAgentDescriptor } from './agentConnection.js'; import type { DesktopAgentConnectionOptions } from './agentConnection.js'; import { loadDesktopConfig, resolveDesktopSpecs, selectDesktopRenderer } from './config.js'; import type { DesktopStage } from './config.js'; +import { createDesktopInputLane, desktopInputEnvironment, requestedDesktopInputLane } from './reporting/lane.js'; import { createDesktopReportSession } from './reporting/session.js'; import { DesktopStopRequested } from './services.js'; import { DesktopSupervisor } from './supervisor.js'; -import { runDesktopWdioSuite } from './wdio/run.js'; +import { assertDesktopSuiteSupportsInput, runDesktopWdioSuite } from './wdio/run.js'; type AgentFlags = { agent?: boolean; descriptor?: string; root?: string }; @@ -53,7 +55,13 @@ export async function runDesktopTestCli(argv = process.argv): Promise { .command('test') .requiredOption('--suite ', 'configured suite') .option('--list', 'resolve exact spec groups without launching') - .action((options: { suite: string; list?: boolean }) => execute('test', options)); + .addOption( + new Option( + '--input ', + `run one input lane: accessible (keyboard, physical click and wheel masked) or physical; falls back to ${desktopInputEnvironment}`, + ).choices([...desktopInputLanes]), + ) + .action((options: { suite: string; list?: boolean; input?: string }) => execute('test', options)); let reporterMode: ResolvedDesktopReporterMode = 'plain'; let reporterResolutionError: unknown; try { @@ -82,7 +90,7 @@ export async function runDesktopTestCli(argv = process.argv): Promise { async function execute( action: DesktopStage | 'build-driver' | 'serve' | 'test', - test?: { suite: string; list?: boolean }, + test?: { suite: string; list?: boolean; input?: string }, agentFlags: AgentFlags = {}, ): Promise { const options = program.opts<{ @@ -118,6 +126,11 @@ export async function runDesktopTestCli(argv = process.argv): Promise { if (options.paper) target = selectDesktopRenderer(target, 'paper'); const suite = test ? config.config.suites?.[test.suite] : undefined; if (test && !suite) throw Object.assign(new Error(`Unknown suite "${test.suite}".`), { code: 'config' }); + if (test?.list && test.input !== undefined) { + throw Object.assign(new TypeError('--list resolves spec files, which do not depend on the input lane; omit --input.'), { + code: 'config', + }); + } if (suite?.kind === 'wdio') { const specs = resolveDesktopSpecs(suite, target.endpoint, config.projectRoot); if (test?.list) { @@ -131,6 +144,12 @@ export async function runDesktopTestCli(argv = process.argv): Promise { return; } } else if (test?.list) throw Object.assign(new Error('--list currently requires a standalone WDIO suite.'), { code: 'config' }); + // Only `test` has a lane. The environment stands in for the flag, but never for `--list`, which returned above. + const requestedLane = test + ? requestedDesktopInputLane({ flag: test.input, environment: process.env[desktopInputEnvironment] }) + : undefined; + const input = requestedLane?.input; + if (suite?.kind === 'wdio') assertDesktopSuiteSupportsInput(test!.suite, suite, input); if (!agentFlags.agent && (agentFlags.descriptor !== undefined || agentFlags.root !== undefined)) throw Object.assign(new TypeError('--descriptor and --root require --agent.'), { code: 'config' }); // An integration serves through its own lifecycle, so it must publish the agent connection itself. @@ -156,6 +175,7 @@ export async function runDesktopTestCli(argv = process.argv): Promise { kind: action === 'test' ? 'tests' : 'lifecycle', optional: suite?.kind === 'wdio' && suite.optional, context: { endpoint: target.endpoint, renderer: target.host.renderer }, + ...(requestedLane ? { lane: createDesktopInputLane(requestedLane.input, requestedLane.source) } : {}), reporterMode, eventLevel, suppressFinalOutput: action === 'serve' ? () => ready : false, @@ -169,6 +189,7 @@ export async function runDesktopTestCli(argv = process.argv): Promise { reportSession: session, integration: config.config.integration, ...(agent ? { agent } : {}), + ...(input ? { input } : {}), }); session.onStop?.(() => owner.services.requestStop()); const interrupt = () => owner.services.requestStop(); @@ -196,6 +217,7 @@ export async function runDesktopTestCli(argv = process.argv): Promise { supervisor: owner, action, suite: test?.suite, + ...(input ? { input } : {}), ...(agent ? { agent } : {}), ...(action === 'serve' ? { reporterMode, onReady: markReady } : {}), }), @@ -219,7 +241,7 @@ export async function runDesktopTestCli(argv = process.argv): Promise { ); markReady(); await owner.services.guard(new Promise(() => undefined)); - } else if (suite?.kind === 'wdio') await runDesktopWdioSuite(owner, config, test!.suite); + } else if (suite?.kind === 'wdio') await runDesktopWdioSuite(owner, config, test!.suite, input); else throw new Error('The imported integration does not implement this suite.'); } } catch (error) { diff --git a/packages/agentic/desktop-test/src/config.ts b/packages/agentic/desktop-test/src/config.ts index edf975b6bb..79fcd0ee85 100644 --- a/packages/agentic/desktop-test/src/config.ts +++ b/packages/agentic/desktop-test/src/config.ts @@ -4,6 +4,10 @@ import { fileURLToPath, pathToFileURL } from 'node:url'; import { createHash } from 'node:crypto'; import type { DesktopEndpoint, DesktopRenderer, NativeDesktopApplicationDescriptor } from '@fluentui-react-native/desktop-driver'; +// The leaf subpath, not `/authoring`: a config file is also loaded from source through the CommonJS prebuild fallback, +// which cannot resolve the `.js` specifiers inside the authoring index. +import { resolveDesktopScopeDomains, resolveDesktopScopeValues } from '@fluentui-react-native/desktop-driver/authoring/input'; +import type { DesktopScopeDomains, DesktopScopeValues } from '@fluentui-react-native/desktop-driver/authoring/input'; import type { DesktopReporterMode } from '@fluentui-react-native/desktop-driver/cli/reporter'; import type { DesktopIntegration } from './integration.ts'; @@ -35,6 +39,11 @@ export type DesktopTargetConfig = { services?: readonly { id: string; command: DesktopCommand; readyUrl?: string }[]; launchTimeoutMs?: number; nativeDriver?: { cacheRoot?: string; helperPath?: string; installRoot?: string }; + /** + * The value this target holds for each custom `testScopes` key a test may scope on, such as a product flavor. + * `rn.renderer` is not set here; it comes from `host.renderer`. + */ + testScope?: Readonly>; }; export type DesktopTargetReference = { use: { config: string; target: string } }; @@ -68,6 +77,11 @@ export type DesktopConfig = { suites?: Readonly>; integration?: DesktopIntegration; reporting?: DesktopReportingConfig; + /** + * Custom scope domains: each key lists the values a test may scope on, and every target sets its own in `testScope`. + * The `rn.*` keys belong to the framework, so a consumer that is not React Native simply declares its own. + */ + testScopes?: Readonly>; }; export type ResolvedDesktopTarget = DesktopTargetConfig & { @@ -84,6 +98,8 @@ export type ResolvedDesktopConfig = { projectRoot: string; targets: Readonly>; reporting: DesktopReportingConfig & { artifactsRoot: string }; + /** The built-in scope domains plus this config's own. */ + testScopes: DesktopScopeDomains; }; const configNames = ['desktop.config.mts', 'desktop.config.ts', 'desktop.config.mjs', 'desktop.config.js', 'desktop.config.cjs']; @@ -98,9 +114,10 @@ export function defineDesktopConfig(config: T): T { export function validateDesktopConfig(value: unknown): asserts value is DesktopConfig { const config = object(value, 'desktop config'); - keys(config, ['schemaVersion', 'projectRoot', 'targets', 'suites', 'integration', 'reporting'], 'desktop config'); + keys(config, ['schemaVersion', 'projectRoot', 'targets', 'suites', 'integration', 'reporting', 'testScopes'], 'desktop config'); if (config.schemaVersion !== 1) throw new TypeError('Desktop config schemaVersion must be 1.'); if (config.projectRoot !== undefined && !(config.projectRoot instanceof URL)) text(config.projectRoot, 'projectRoot'); + const scopeDomains = resolveDesktopScopeDomains(config.testScopes); const targets = object(config.targets, 'targets'); if (Object.keys(targets).length === 0) throw new TypeError('Desktop config must declare at least one target.'); for (const [id, entry] of Object.entries(targets)) { @@ -114,8 +131,9 @@ export function validateDesktopConfig(value: unknown): asserts value is DesktopC text(reference.target, `${id}.use.target`); continue; } - keys(target, ['endpoint', 'host', 'readiness', 'commands', 'services', 'launchTimeoutMs', 'nativeDriver'], `target ${id}`); + keys(target, ['endpoint', 'host', 'readiness', 'commands', 'services', 'launchTimeoutMs', 'nativeDriver', 'testScope'], `target ${id}`); oneOf(target.endpoint ?? id, endpoints, `${id}.endpoint`); + if (target.testScope !== undefined) validateTestScope(target.testScope, scopeDomains, id); const host = object(target.host, `${id}.host`); keys(host, ['kind', 'application', 'ownership', 'renderer', 'appConfigPath', 'component', 'windowTitle', 'bundleRoot'], `${id}.host`); oneOf(host.kind, ['installed', 'react-native-test-app', 'rex'], `${id}.host.kind`); @@ -241,6 +259,8 @@ export async function loadDesktopConfig(configPath?: string, cwd = process.cwd() } return pending; }; + const entry = await read(filename); + const testScopes = resolveDesktopScopeDomains(entry.config.testScopes); const resolve = async (filename: string, id: string, ancestry: readonly string[]): Promise => { const { config, root, file } = await read(filename); const key = `${file}#${id}`; @@ -248,6 +268,8 @@ export async function loadDesktopConfig(configPath?: string, cwd = process.cwd() const target = config.targets[id]; if (!target) throw new Error(`Unknown desktop target "${id}" in ${file}.`); if ('use' in target) return resolve(path.resolve(path.dirname(file), target.use.config), target.use.target, [...ancestry, key]); + // A referenced target was checked against its own config's domains; the config that uses it must declare them too. + if (target.testScope) resolveDesktopScopeValues(target.testScope, testScopes, `${id}.testScope`); const endpoint = target.endpoint ?? (id as DesktopEndpoint); const host = { ...target.host, @@ -276,17 +298,38 @@ export async function loadDesktopConfig(configPath?: string, cwd = process.cwd() : undefined, }); }; - const { config, root, file } = await read(filename); - const targets = Object.fromEntries(await Promise.all(Object.keys(config.targets).map(async (id) => [id, await resolve(file, id, [])]))); + const targets = Object.fromEntries( + await Promise.all(Object.keys(entry.config.targets).map(async (id) => [id, await resolve(entry.file, id, [])])), + ); return { - config, - configPath: file, - projectRoot: root, + config: entry.config, + configPath: entry.file, + projectRoot: entry.root, targets: Object.freeze(targets), - reporting: { ...config.reporting, artifactsRoot: path.resolve(root, config.reporting?.artifactsRoot ?? 'artifacts/desktop') }, + reporting: { + ...entry.config.reporting, + artifactsRoot: path.resolve(entry.root, entry.config.reporting?.artifactsRoot ?? 'artifacts/desktop'), + }, + testScopes, }; } +/** + * The values a target holds for the scope domains: its own `testScope` plus the renderer its host runs. A caller that + * knows the renderer better than the host configuration, such as a Storybook run switched to Paper, passes it. + */ +export function resolveDesktopTestScope( + domains: DesktopScopeDomains, + target?: Pick, + renderer: DesktopRenderer | undefined = target?.host.renderer, +): DesktopScopeValues { + return resolveDesktopScopeValues( + { ...target?.testScope, ...(renderer ? { 'rn.renderer': renderer } : {}) }, + domains, + target ? `${target.id}.testScope` : 'testScope', + ); +} + export function selectDesktopRenderer(target: ResolvedDesktopTarget, renderer: DesktopRenderer): ResolvedDesktopTarget { if (target.endpoint !== 'macos' || target.host.kind !== 'react-native-test-app') throw new Error('Renderer switching requires a macOS RN Test App target; regenerate with prep after switching.'); @@ -327,6 +370,14 @@ function resolvePaths(root: string, settings: object, fields: readonly string[]) ); } +function validateTestScope(value: unknown, domains: DesktopScopeDomains, id: string): void { + const scope = object(value, `${id}.testScope`); + for (const key of Object.keys(scope)) { + if (key.startsWith('rn.')) throw new TypeError(`${id}.testScope cannot set "${key}"; the rn.* values come from ${id}.host.`); + } + resolveDesktopScopeValues(scope, domains, `${id}.testScope`); +} + function validateApplication(value: unknown, context: string): void { const application = object(value, context); keys( diff --git a/packages/agentic/desktop-test/src/contracts.contract.cjs b/packages/agentic/desktop-test/src/contracts.contract.cjs index 7fef3c4e35..7182e5f8f1 100644 --- a/packages/agentic/desktop-test/src/contracts.contract.cjs +++ b/packages/agentic/desktop-test/src/contracts.contract.cjs @@ -49,7 +49,7 @@ async function main() { const { DesktopRunReporter } = await import(lib('reporting/collector.js')); const { NodeDesktopCommandRunner } = await import(lib('commands.js')); const { DesktopServiceScope, DesktopStopRequested } = await import(lib('services.js')); - const { loadDesktopConfig, defineDesktopConfig, resolveDesktopSpecs } = await import(lib('config.js')); + const { loadDesktopConfig, defineDesktopConfig, resolveDesktopSpecs, resolveDesktopTestScope } = await import(lib('config.js')); // Config resolution canonicalizes paths; macOS temporary directories are symlinked (/var -> /private/var). const root = fs.realpathSync.native(fs.mkdtempSync(path.join(os.tmpdir(), 'desktop-harness-contract-'))); const out = capture(); @@ -92,6 +92,43 @@ async function main() { 'export default {schemaVersion:1, targets:{win32:{use:{config:"cycle.mjs",target:"win32"}}}}', ); await assert.rejects(loadDesktopConfig(path.join(root, 'cycle.mjs')), /cycle/); + const flavored = (testScopes, testScope, host = app.targets.win32.host) => ({ + ...app, + ...(testScopes ? { testScopes } : {}), + targets: { win32: { host, ...(testScope ? { testScope } : {}) } }, + }); + const flavors = { 'product.flavor': ['alpha', 'beta'] }; + defineDesktopConfig(flavored(flavors, { 'product.flavor': 'beta' })); + assert.throws(() => defineDesktopConfig(flavored({ 'rn.renderer': ['x'] })), /reserved/); + assert.throws(() => defineDesktopConfig(flavored(flavors, { 'product.flavor': 'gamma' })), /must be one of: alpha, beta/); + assert.throws(() => defineDesktopConfig(flavored(flavors, { unknown: 'x' })), /unknown scope key "unknown"/); + assert.throws(() => defineDesktopConfig(flavored(flavors, { 'rn.renderer': 'paper' })), /cannot set "rn.renderer"/); + fs.mkdirSync(path.join(root, 'flavor')); + const paperHost = { ...app.targets.win32.host, renderer: 'paper' }; + fs.writeFileSync( + path.join(root, 'flavor', 'desktop.config.mjs'), + `export default ${JSON.stringify(flavored(flavors, { 'product.flavor': 'beta' }, paperHost))};`, + ); + const flavor = await loadDesktopConfig(path.join(root, 'flavor', 'desktop.config.mjs')); + assert.deepEqual({ ...flavor.testScopes }, { 'rn.renderer': ['fabric', 'paper'], ...flavors }); + assert.deepEqual( + { ...resolveDesktopTestScope(flavor.testScopes, flavor.targets.win32) }, + { 'product.flavor': 'beta', 'rn.renderer': 'paper' }, + ); + assert.deepEqual( + { ...resolveDesktopTestScope(flavor.testScopes, flavor.targets.win32, 'fabric') }, + { 'product.flavor': 'beta', 'rn.renderer': 'fabric' }, + ); + assert.deepEqual( + { ...resolveDesktopTestScope(direct.testScopes, direct.targets.win32) }, + {}, + 'no renderer is derived without a host renderer', + ); + fs.writeFileSync( + path.join(root, 'flavor', 'consumer.mjs'), + `export default ${JSON.stringify({ schemaVersion: 1, targets: { win32: { use: { config: './desktop.config.mjs', target: 'win32' } } } })};`, + ); + await assert.rejects(loadDesktopConfig(path.join(root, 'flavor', 'consumer.mjs')), /unknown scope key "product.flavor"/); fs.writeFileSync(path.join(root, 'one.spec.mjs'), ''); fs.writeFileSync(path.join(root, 'two.spec.mjs'), ''); assert.deepEqual(resolveDesktopSpecs({ kind: 'wdio', framework: 'jasmine', specs: [['*.spec.mjs']] }, 'win32', root), [ @@ -1188,11 +1225,110 @@ async function main() { logs.some((log) => log.includes('serve --agent requires an integration that supports agents; \\"fixture\\" does not.')), logs.join('\n'), ); + } else if (mode === 'lane-cli') { + // Each invocation is its own process with no inherited collector, so its run result records the lane it was given. + const configPath = path.join(root, 'desktop.config.mjs'); + fs.writeFileSync(path.join(root, 'spec.mjs'), 'export {};\n'); + fs.writeFileSync( + configPath, + [ + "import fs from 'node:fs';", + 'export default {', + ' schemaVersion: 1,', + " targets: { win32: { host: { kind: 'installed', application: { executablePath: 'not-launched.exe' } } } },", + ' suites: {', + " csf: { kind: 'csf-wdio' },", + " jasmine: { kind: 'wdio', framework: 'jasmine', specs: ['spec.mjs'] },", + " mocha: { kind: 'wdio', framework: 'mocha', specs: ['spec.mjs'] },", + ' },', + ' reporting: { artifactsRoot: process.env.FIXTURE_ARTIFACTS },', + ' integration: {', + " name: 'fixture',", + ' validate() {},', + ' async run({ action, suite, input, supervisor }) {', + ' fs.writeFileSync(process.env.FIXTURE_MARKER, JSON.stringify({ action, suite, input: input ?? null }));', + " supervisor.reporter.emit({ kind: 'test-result', payload: { testId: 'fixture', title: 'fixture', attempt: 0, status: 'passed', durationMs: 1 } });", + ' },', + ' },', + '};', + '', + ].join('\n'), + ); + const inherited = { ...process.env }; + for (const name of ['DESKTOP_REPORT_CONTEXT', 'DESKTOP_INPUT', 'DESKTOP_REPORTER', 'DESKTOP_EVENTS']) delete inherited[name]; + let invocations = 0; + const invoke = (args, environment = {}) => { + const index = invocations++; + const marker = path.join(root, `marker-${index}.json`); + const artifacts = path.join(root, `artifacts-${index}`); + const child = spawnSync( + process.execPath, + [path.join(packageRoot, 'config', 'cli.cjs'), '--config', configPath, '--target', 'win32', '--reporter', 'json', ...args], + { cwd: root, encoding: 'utf8', env: { ...inherited, FIXTURE_MARKER: marker, FIXTURE_ARTIFACTS: artifacts, ...environment } }, + ); + const runs = fs.existsSync(artifacts) ? fs.readdirSync(artifacts) : []; + return { + status: child.status, + diagnostics: `${child.stdout}\n${child.stderr}`, + marker: fs.existsSync(marker) ? JSON.parse(fs.readFileSync(marker, 'utf8')) : undefined, + result: runs.length ? JSON.parse(fs.readFileSync(path.join(artifacts, runs[0], 'run.json'), 'utf8')) : undefined, + error: child.stdout.startsWith('{') ? JSON.parse(child.stdout.split('\n')[0]).error : undefined, + }; + }; + const ran = (run, input, lane) => { + assert.equal(run.status, 0, run.diagnostics); + assert.deepEqual(run.marker, { action: 'test', suite: 'csf', input }); + assert.deepEqual(run.result.lane, lane); + }; + ran(invoke(['test', '--suite', 'csf', '--input', 'accessible']), 'accessible', { + input: 'accessible', + source: 'option', + masking: 'host-masked', + policyEnforced: false, + }); + ran(invoke(['test', '--suite', 'csf'], { DESKTOP_INPUT: 'physical' }), 'physical', { + input: 'physical', + source: 'environment', + policyEnforced: false, + }); + ran(invoke(['test', '--suite', 'csf', '--input', 'accessible'], { DESKTOP_INPUT: 'physical' }), 'accessible', { + input: 'accessible', + source: 'option', + masking: 'host-masked', + policyEnforced: false, + }); + const unlaned = invoke(['test', '--suite', 'csf']); + assert.equal(unlaned.status, 0, unlaned.diagnostics); + assert.deepEqual(unlaned.marker, { action: 'test', suite: 'csf', input: null }); + assert.equal('lane' in unlaned.result, false, 'a run with no lane must not claim one'); + const refused = (run, pattern) => { + assert.equal(run.status, 1, run.diagnostics); + assert.equal(run.marker, undefined, 'a refused lane must not start the run'); + assert.equal(run.result, undefined, 'a refused lane must not open a result'); + assert.equal(run.error?.code, 'config', run.diagnostics); + assert.match(run.error.message, pattern); + }; + refused(invoke(['test', '--suite', 'jasmine', '--list', '--input', 'accessible']), /--list resolves spec files/); + refused( + invoke(['test', '--suite', 'mocha', '--input', 'accessible']), + /--input accessible requires a jasmine suite; "mocha" uses mocha/, + ); + refused(invoke(['test', '--suite', 'mocha'], { DESKTOP_INPUT: 'physical' }), /--input physical requires a jasmine suite/); + refused(invoke(['test', '--suite', 'csf'], { DESKTOP_INPUT: 'fast' }), /DESKTOP_INPUT must be accessible or physical/); + const listed = invoke(['test', '--suite', 'jasmine', '--list'], { DESKTOP_INPUT: 'accessible' }); + assert.equal(listed.status, 0, `--list resolves files and ignores the environment lane\n${listed.diagnostics}`); } else { const { DesktopSupervisor } = await import(lib('supervisor.js')); const { runDesktopWdioSuite } = await import(lib('wdio/run.js')); + const { createDesktopInputLane } = await import(lib('reporting/lane.js')); + const { resolveDesktopScopeDomains } = await import('@fluentui-react-native/desktop-driver/authoring'); const { FakeDesktopHost } = await import('@fluentui-react-native/desktop-driver/testing'); - const run = reporter({ context: { endpoint: 'win32', renderer: 'paper' } }); + const laneMode = mode.startsWith('lane-'); + const retried = mode === 'retry' || mode === 'lane-physical'; + const lane = laneMode ? createDesktopInputLane(mode === 'lane-physical' ? 'physical' : 'accessible', 'option') : undefined; + // The lane fixtures import the Jasmine adapter by file URL from the environment, so no value is spliced into source. + if (laneMode) process.env.DESKTOP_JASMINE_URL = lib('jasmine/index.js'); + const run = reporter({ context: { endpoint: 'win32', renderer: 'paper' }, ...(lane ? { lane } : {}) }); const fixture = path.join(root, mode === 'typescript' ? 'spec.ts' : 'spec.mjs'); const second = path.join(root, 'second.mjs'); let source; @@ -1213,6 +1349,51 @@ async function main() { "describe('hook fixture',()=>{beforeAll(()=>{throw new Error('original setup failure');});it('does not execute',()=>{throw new Error('body must not run');});});"; } else if (mode === 'crash') { source = 'process.exit(17);'; + } else if (mode === 'lane-physical') { + source = [ + "import fs from 'node:fs';", + 'const { desktopIt, desktopDescribe } = await import(process.env.DESKTOP_JASMINE_URL);', + "const marker = new URL('./attempt', import.meta.url);", + 'const mustNotRun = (what) => () => { throw new Error(`${what} must not run`); };', + "describe('physical lane fixture', () => {", + " beforeAll(() => { globalThis.sharedHook = 'ran'; });", + " desktopIt('runs when nothing is declared', () => { expect(globalThis.sharedHook).toBe('ran'); });", + " desktopIt('clicks as the lane does', { requires: ['click'] }, async () => { await (await browser.$('~button-primary')).click(); });", + " desktopIt('needs the keyboard', { requires: ['keyboard'] }, () => {});", + " desktopIt('fails once, then retries', () => { if (!fs.existsSync(marker)) { fs.writeFileSync(marker, 'attempt'); throw new Error('first attempt'); } });", + " desktopIt('accessible only', { requires: ['accessibility-click'] }, mustNotRun('an input-excluded case'));", + " desktopIt('macOS only', { platforms: ['macos'] }, mustNotRun('a platform-excluded case'));", + " desktopIt('fabric only', { scope: { 'rn.renderer': ['fabric'] } }, mustNotRun('a scope-excluded case'));", + " desktopIt('declared gap', { unsupported: { platforms: ['win32'], reason: 'Declared gap.' } }, mustNotRun('a declared gap'));", + '});', + "desktopDescribe('excluded group', { platforms: ['macos'] }, () => {", + " beforeAll(mustNotRun('the hook of an excluded group'));", + " beforeEach(mustNotRun('the hook of an excluded group'));", + " desktopIt('inside an excluded group', mustNotRun('a platform-excluded case'));", + '});', + ].join('\n'); + } else if (mode === 'lane-accessible') { + source = [ + 'const { desktopIt } = await import(process.env.DESKTOP_JASMINE_URL);', + 'const mustNotRun = (what) => () => { throw new Error(`${what} must not run`); };', + "describe('accessible lane fixture', () => {", + " desktopIt('runs when nothing is declared', () => {});", + " desktopIt('clicks as the lane does', { requires: ['click'] }, async () => { await (await browser.$('~button-primary')).click(); });", + " desktopIt('clicks accessibly', { requires: ['accessibility-click'] }, async () => { await (await browser.$('~button-primary')).click(); });", + " desktopIt('physical only', { requires: ['keyboard'] }, mustNotRun('an input-excluded case'));", + " desktopIt('needs what the host lacks', { requires: ['accessibility-actions'] }, mustNotRun('a case the host cannot support'));", + '});', + ].join('\n'); + } else if (mode === 'lane-blocked') { + source = [ + "import fs from 'node:fs';", + 'const { desktopIt } = await import(process.env.DESKTOP_JASMINE_URL);', + "const ran = new URL('./ran', import.meta.url);", + "describe('blocked lane fixture', () => {", + " desktopIt('declared', () => { fs.writeFileSync(ran, 'ran'); });", + " it('undeclared', () => { fs.writeFileSync(ran, 'ran'); });", + '});', + ].join('\n'); } else if (mode === 'bail') { source = "describe('bail fixture',()=>{it('fails first',()=>{throw new Error('original bail failure');});});"; fs.writeFileSync( @@ -1236,7 +1417,11 @@ async function main() { projectRoot: packageRoot, configPath: path.join(root, 'desktop.config.mjs'), }; - const host = new FakeDesktopHost({ endpoint: 'win32' }); + // The accessible lane masks keyboard, physical click and wheel on the host. Only `lane-blocked` leaves them on. + const host = new FakeDesktopHost({ + endpoint: 'win32', + ...(mode === 'lane-accessible' ? { features: { keyboard: false, physicalClick: false, wheel: false } } : {}), + }); let launches = 0; const launch = host.launch.bind(host); host.launch = (...args) => { @@ -1259,7 +1444,7 @@ async function main() { kind: 'wdio', framework: 'jasmine', specs: mode === 'bail' ? [fixture, second] : [[fixture, ...(mode === 'wdio' ? [second] : [])]], - specFileRetries: mode === 'retry' ? 1 : 0, + specFileRetries: retried ? 1 : 0, }, }, }, @@ -1267,6 +1452,7 @@ async function main() { configPath: target.configPath, targets: { win32: target }, reporting: { artifactsRoot: root }, + testScopes: resolveDesktopScopeDomains(), }; if (mode === 'config-hook-failure') config.config.suites.fixture.readiness = { kind: 'native', testId: 'missing-readiness-marker', timeoutMs: 50 }; @@ -1294,7 +1480,7 @@ async function main() { }); await assert.rejects(duplicate.start(), /Another desktop workload/); try { - await runDesktopWdioSuite(supervisor, config, 'fixture'); + await runDesktopWdioSuite(supervisor, config, 'fixture', lane?.input); } catch (error) { failure = error; run.fail(error); @@ -1303,7 +1489,7 @@ async function main() { await supervisor.close(); } const result = await run.finish(); - if (['hook-failure', 'config-hook-failure', 'config-load-failure', 'crash', 'bail'].includes(mode)) { + if (['hook-failure', 'config-hook-failure', 'config-load-failure', 'crash', 'bail', 'lane-blocked'].includes(mode)) { assert(failure, `hook failure must fail launcher\n${JSON.stringify(result)}\n${err.text()}\n${out.text()}`); assert.equal(result.status, 'failed'); assert(!out.text().includes('PASSED WDIO fixture'), out.text()); @@ -1322,15 +1508,49 @@ async function main() { result.work.some(({ status }) => status === 'not-run'), JSON.stringify(result), ); + if (mode === 'lane-blocked') { + // An unmasked accessible host and an undeclared spec are both reported, and neither spec ran. + const text = JSON.stringify(result); + assert(text.includes('needs a host that reports keyboard, physicalClick, wheel disabled'), text); + assert(text.includes('not declared with desktopIt or desktopDescribe'), text); + assert(!fs.existsSync(path.join(root, 'ran')), 'a blocked lane must not run any spec'); + assert.equal(result.counts.executed, 0, text); + assert.equal(result.counts.skipped, 0, 'a blocked lane reports its fault, not every spec it disabled'); + } + } else if (mode === 'lane-accessible') { + const text = JSON.stringify(result); + // The host cannot support one case. A lane run holds that against the run instead of quietly skipping it. + assert(failure && /incomplete/.test(failure.message), `${failure?.stack}\n${text}\n${err.text()}`); + assert.notEqual(result.status, 'passed', text); + assert.equal(launches, 1); + assert.equal(result.counts.passed, 3, text); + assert.equal(result.counts.failed, 0, text); + assert.equal(result.counts.unsupported, 1, text); + assert.equal(result.counts.declaredUnsupported, 0, text); + assert.deepEqual(result.selection.excluded, { platform: 0, scope: 0, input: 1 }, text); + assert.deepEqual(result.lane, { input: 'accessible', source: 'option', masking: 'host-masked', policyEnforced: false }); + const clicks = host.actions.filter((action) => action.type === 'click'); + assert(clicks.length === 2 && clicks.every((action) => action.mode === 'accessibility'), JSON.stringify(host.actions)); } else { assert.equal(failure, undefined, `${failure?.stack}\n${err.text()}\n${out.text()}`); assert.equal(result.status, 'passed', JSON.stringify(result)); - assert.equal(launches, mode === 'retry' ? 2 : 1, 'WDIO must own exactly one session per grouped attempt'); - assert.equal(result.counts.flaky, mode === 'retry' ? 1 : 0, JSON.stringify(result)); + assert.equal(launches, retried ? 2 : 1, 'WDIO must own exactly one session per grouped attempt'); + assert.equal(result.counts.flaky, retried ? 1 : 0, JSON.stringify(result)); if (mode === 'wdio') { assert.equal(result.counts.passed, 2); assert.equal(result.counts.skipped, 1); } + if (mode === 'lane-physical') { + const text = JSON.stringify(result); + assert.equal(result.counts.passed, 4, text); + assert.equal(result.counts.unsupported, 1, text); + assert.equal(result.counts.declaredUnsupported, 1, text); + // Both attempts announced the same exclusions, and a retried group still counts them once. + assert.deepEqual(result.selection.excluded, { platform: 2, scope: 1, input: 1 }, text); + assert.deepEqual(result.lane, { input: 'physical', source: 'option', policyEnforced: false }); + const clicks = host.actions.filter((action) => action.type === 'click'); + assert(clicks.length > 0 && clicks.every((action) => action.mode === 'physical'), JSON.stringify(host.actions)); + } } assert(!fs.existsSync(path.join(root, 'locks', 'interactive.lock'))); } diff --git a/packages/agentic/desktop-test/src/contracts.test.ts b/packages/agentic/desktop-test/src/contracts.test.ts index 559708c809..d666a3dcd5 100644 --- a/packages/agentic/desktop-test/src/contracts.test.ts +++ b/packages/agentic/desktop-test/src/contracts.test.ts @@ -25,6 +25,10 @@ describe('neutral desktop harness contracts', () => { 'config-load-failure', 'crash', 'bail', + 'lane-physical', + 'lane-accessible', + 'lane-blocked', + 'lane-cli', ])( '%s contract', async (mode) => { diff --git a/packages/agentic/desktop-test/src/index.ts b/packages/agentic/desktop-test/src/index.ts index 946718dc9b..1b61b3ad75 100644 --- a/packages/agentic/desktop-test/src/index.ts +++ b/packages/agentic/desktop-test/src/index.ts @@ -10,7 +10,7 @@ export type { PublishDesktopAgentConnection, } from './agentConnection.js'; export { DesktopServiceScope, DesktopStopRequested } from './services.js'; -export { defineDesktopConfig, loadDesktopConfig, resolveDesktopSpecs, validateDesktopConfig } from './config.js'; +export { defineDesktopConfig, loadDesktopConfig, resolveDesktopSpecs, resolveDesktopTestScope, validateDesktopConfig } from './config.js'; export type { DesktopConfig, DesktopTargetConfig, DesktopWdioSuite, ResolvedDesktopConfig, ResolvedDesktopTarget } from './config.js'; export type { DesktopIntegration } from './integration.js'; export { desktopStageCommands, resolveDesktopApplication } from './adapters.js'; diff --git a/packages/agentic/desktop-test/src/integration.ts b/packages/agentic/desktop-test/src/integration.ts index cdb74ede46..35fc5021a6 100644 --- a/packages/agentic/desktop-test/src/integration.ts +++ b/packages/agentic/desktop-test/src/integration.ts @@ -1,4 +1,5 @@ import type { DesktopHost, DesktopTarget } from '@fluentui-react-native/desktop-driver'; +import type { DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; import type { ResolvedDesktopReporterMode } from '@fluentui-react-native/desktop-driver/cli/reporter'; import type { DesktopAgentConnectionOptions } from './agentConnection.js'; import type { DesktopStage, ResolvedDesktopConfig, ResolvedDesktopTarget } from './config.js'; @@ -30,6 +31,8 @@ export type DesktopIntegration = { supervisor: DesktopSupervisor; action: DesktopStage | 'test' | 'serve'; suite?: string; + /** Set only for `test`, when the run executes one input lane. The integration owns masking its own driver. */ + input?: DesktopInputLane; /** Set only for `serve --agent`, and only for an integration that `supportsAgent`. */ agent?: Pick; /** diff --git a/packages/agentic/desktop-test/src/jasmine/adapter.test.ts b/packages/agentic/desktop-test/src/jasmine/adapter.test.ts new file mode 100644 index 0000000000..5b0764a76c --- /dev/null +++ b/packages/agentic/desktop-test/src/jasmine/adapter.test.ts @@ -0,0 +1,208 @@ +import { createDesktopJasmine } from './adapter.js'; +import type { DesktopJasmineApi } from './adapter.js'; +import { createDesktopJasmineRegistry, desktopSpecIdentifier, excludedSpecCounts } from './registry.js'; +import { desktopTestSelectionEnvironment, serializeDesktopTestSelection } from './selection.js'; + +type DesktopBody = (done?: (error?: unknown) => void) => unknown; +type Registered = { kind: 'describe' | 'it' | 'xit'; name: string; path: string[]; body?: DesktopBody }; + +/** Registers what a Jasmine would, and hands back the metadata `it` and `xit` return. */ +function fakeJasmine(options: { identify?: boolean } = {}) { + const registered: Registered[] = []; + const path: string[] = []; + let next = 0; + const spec = (kind: 'it' | 'xit', name: string, body?: DesktopBody) => { + registered.push({ kind, name, path: [...path], body }); + return options.identify === false ? undefined : { id: `spec${next++}`, description: name }; + }; + const api: DesktopJasmineApi = { + describe(name, body) { + registered.push({ kind: 'describe', name, path: [...path] }); + path.push(name); + try { + body.call({}); + } finally { + path.pop(); + } + }, + it: (name, body) => spec('it', name, body), + xit: (name) => spec('xit', name), + }; + return { api, registered }; +} + +function laneEnvironment(lane?: 'accessible' | 'physical', extra: { testScopes?: unknown; values?: Record } = {}) { + return { + [desktopTestSelectionEnvironment]: serializeDesktopTestSelection({ + endpoint: 'win32', + ...(lane ? { lane } : {}), + ...(extra.testScopes ? { testScopes: extra.testScopes } : {}), + values: { 'rn.renderer': 'paper', ...extra.values }, + }), + }; +} + +function setup(lane?: 'accessible' | 'physical', extra?: Parameters[1], options?: { identify?: boolean }) { + const registry = createDesktopJasmineRegistry(laneEnvironment(lane, extra)); + const jasmine = fakeJasmine(options); + return { registry, ...jasmine, ...createDesktopJasmine(jasmine.api, registry) }; +} + +const body = () => undefined; + +describe('desktopIt without a selection', () => { + test('registers an ordinary spec, so a runner that does not select keeps working', () => { + const registry = createDesktopJasmineRegistry(); + const { api, registered } = fakeJasmine(); + const { desktopIt } = createDesktopJasmine(api, registry); + desktopIt('plain', body); + desktopIt('physical', { requires: ['keyboard'] }, body); + + expect(registered.map(({ kind, name }) => `${kind}:${name}`)).toEqual(['it:plain', 'it:physical']); + expect(registry.errors).toEqual([]); + }); + + test('still rejects a declaration it cannot read', () => { + const registry = createDesktopJasmineRegistry(); + const { api, registered } = fakeJasmine(); + const { desktopIt } = createDesktopJasmine(api, registry); + desktopIt('bad', { requires: ['telepathy' as never] }, body); + + expect(registered).toHaveLength(1); + expect(() => registered[0].body?.()).toThrow(/unsupported requirement/); + expect(registry.entries.get('badspec0')).toEqual({ kind: 'invalid' }); + }); +}); + +describe('desktopIt in a lane', () => { + test('runs what the lane covers and replaces the rest with placeholders that are counted by reason', () => { + const { desktopIt, registered, registry } = setup('accessible'); + desktopIt('both', body); + desktopIt('accessible only', { requires: ['accessibility-click'] }, body); + desktopIt('physical only', { requires: ['keyboard'] }, body); + desktopIt('macOS only', { platforms: ['macos'] }, body); + desktopIt('fabric only', { scope: { 'rn.renderer': ['fabric'] } }, body); + + expect(registered.map(({ kind, name }) => `${kind}:${name}`)).toEqual([ + 'it:both', + 'it:accessible only', + 'xit:physical only', + 'xit:macOS only', + 'xit:fabric only', + ]); + expect(excludedSpecCounts(registry)).toEqual({ platform: 1, scope: 1, input: 1 }); + expect(registry.entries.get('physical onlyspec2')).toEqual({ kind: 'excluded', reason: 'input' }); + }); + + test('a case is excluded once, for the first reason that applies', () => { + const { desktopIt, registry } = setup('accessible'); + desktopIt('everything', { platforms: ['macos'], scope: { 'rn.renderer': ['fabric'] }, requires: ['keyboard'] }, body); + + expect(excludedSpecCounts(registry)).toEqual({ platform: 1, scope: 0, input: 0 }); + }); + + test('a declared gap becomes a placeholder that carries its reason, not an excluded case', () => { + const { desktopIt, registered, registry } = setup('physical'); + desktopIt('known gap', { unsupported: { platforms: ['win32'], reason: 'Not on win32 yet.' } }, body); + desktopIt('known gap elsewhere', { unsupported: { platforms: ['macos'], reason: 'Not on macOS yet.' } }, body); + + expect(registered.map(({ kind }) => kind)).toEqual(['xit', 'it']); + expect(registry.entries.get('known gapspec0')).toEqual({ kind: 'gap', gap: { kind: 'unsupported', reason: 'Not on win32 yet.' } }); + expect(excludedSpecCounts(registry)).toEqual({ platform: 0, scope: 0, input: 0 }); + }); + + test('an expired quarantine fails that case instead of hiding it', () => { + const { desktopIt, registered, registry } = setup('physical'); + desktopIt('stale', { quarantine: { issue: '#1', reason: 'Flaky.', expires: '2000-01-01' } }, body); + + expect(registered.map(({ kind }) => kind)).toEqual(['it']); + expect(() => registered[0].body?.()).toThrow('Quarantine #1 expired on 2000-01-01.'); + expect(registry.entries.get('stalespec0')).toEqual({ kind: 'invalid' }); + }); + + test('a custom scope key is checked against the domains the config declared', () => { + const extra = { testScopes: { flavor: ['dev', 'prod'] }, values: { flavor: 'dev' } }; + const accepted = setup('physical', extra); + accepted.desktopIt('dev only', { scope: { flavor: ['dev'] } }, body); + accepted.desktopIt('prod only', { scope: { flavor: ['prod'] } }, body); + expect(accepted.registered.map(({ kind }) => kind)).toEqual(['it', 'xit']); + + const unknown = setup('physical'); + unknown.desktopIt('typo', { scope: { flavour: ['dev'] } }, body); + expect(() => unknown.registered[0].body?.()).toThrow(/unknown scope key "flavour"/); + }); + + test('records the spec by the identifier WDIO reports, and faults when Jasmine does not identify it', () => { + expect(desktopSpecIdentifier({ description: 'a case', id: 'spec7' })).toBe('a casespec7'); + expect(desktopSpecIdentifier({ description: 'a case' })).toBeUndefined(); + + const { desktopIt, registry } = setup('physical', undefined, { identify: false }); + desktopIt('anonymous', body); + expect(registry.errors).toEqual([expect.stringContaining('did not identify the spec registered for "anonymous"')]); + }); +}); + +describe('desktopDescribe', () => { + test('adds what a group requires to every case inside it, however deep', () => { + const { desktopDescribe, desktopIt, registered } = setup('accessible'); + desktopDescribe('group', { requires: ['keyboard'] }, () => { + desktopIt('inherits', body); + desktopDescribe('nested', () => desktopIt('also inherits', body)); + }); + desktopIt('outside', body); + + expect(registered.filter(({ kind }) => kind !== 'describe').map(({ kind, name }) => `${kind}:${name}`)).toEqual([ + 'xit:inherits', + 'xit:also inherits', + 'it:outside', + ]); + }); + + test('narrows platforms to the overlap of the group and the case', () => { + const { desktopDescribe, desktopIt, registered } = setup('physical'); + desktopDescribe('group', { platforms: ['!macos'] }, () => { + desktopIt('keeps win32', { platforms: ['macos', 'win32'] }, body); + desktopIt('loses win32', { platforms: ['macos', 'windows'] }, body); + }); + + expect(registered.filter(({ kind }) => kind !== 'describe').map(({ kind, name }) => `${kind}:${name}`)).toEqual([ + 'it:keeps win32', + 'xit:loses win32', + ]); + }); + + test('a case that could never run inside its group is a failing spec, not a silent exclusion', () => { + const { desktopDescribe, desktopIt, registered } = setup('physical'); + desktopDescribe('group', { platforms: ['macos'] }, () => { + desktopIt('never', { platforms: ['win32'] }, body); + }); + + const specs = registered.filter(({ kind }) => kind === 'it'); + expect(specs).toHaveLength(1); + expect(() => specs[0].body?.()).toThrow(/do not overlap the enclosing group's, so it could never run/); + }); + + test('a group cannot declare a gap, so its body does not run and one spec reports why', () => { + const { desktopDescribe, registered } = setup('physical'); + const inner = jest.fn(); + desktopDescribe('group', { unsupported: { platforms: ['win32'], reason: 'No.' } } as never, inner); + + expect(inner).not.toHaveBeenCalled(); + const specs = registered.filter(({ kind }) => kind === 'it'); + expect(specs).toHaveLength(1); + expect(() => specs[0].body?.()).toThrow(/cannot declare unsupported or quarantine/); + }); + + test('a group that throws still releases its options for the next group', () => { + const { desktopDescribe, desktopIt, registered, registry } = setup('accessible'); + expect(() => + desktopDescribe('throws', { requires: ['keyboard'] }, () => { + throw new Error('boom'); + }), + ).toThrow('boom'); + expect(registry.groups).toEqual([]); + + desktopIt('after', body); + expect(registered.filter(({ kind }) => kind === 'it' || kind === 'xit').map(({ kind }) => kind)).toEqual(['it']); + }); +}); diff --git a/packages/agentic/desktop-test/src/jasmine/adapter.ts b/packages/agentic/desktop-test/src/jasmine/adapter.ts new file mode 100644 index 0000000000..9139684e15 --- /dev/null +++ b/packages/agentic/desktop-test/src/jasmine/adapter.ts @@ -0,0 +1,103 @@ +import { + decideDesktopCase, + mergeDesktopCaseOptions, + validateDesktopCaseOptions, + validateDesktopGroupOptions, +} from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopCaseOptions, ResolvedDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; +import { desktopSpecIdentifier } from './registry.js'; +import type { DesktopJasmineRegistry, DesktopSpecEntry } from './registry.js'; + +export type DesktopSpecBody = (done: (error?: unknown) => void) => unknown; +export type DesktopGroupBody = (this: unknown) => void; +/** A group scopes the cases inside it. A gap describes one case, so a group cannot declare one. */ +export type DesktopGroupOptions = Omit; + +/** The part of the Jasmine globals the adapter registers specs through. */ +export type DesktopJasmineApi = { + describe(name: string, body: DesktopGroupBody): unknown; + it(name: string, body: DesktopSpecBody): unknown; + xit(name: string): unknown; +}; + +/** + * Declares tests with their input modality and the places they run, and registers them with Jasmine. + * + * `desktopIt` and `desktopDescribe` never throw for a bad declaration: Jasmine only warns when a spec file fails to load, + * so a throw would silently drop every test after it. A bad declaration becomes a failing spec that reports the problem. + */ +export function createDesktopJasmine(api: DesktopJasmineApi, registry: DesktopJasmineRegistry) { + function record(spec: unknown, name: string, entry: DesktopSpecEntry): void { + const identifier = desktopSpecIdentifier(spec as { description?: unknown; id?: unknown } | undefined); + if (identifier === undefined) { + registry.errors.push(`Jasmine did not identify the spec registered for "${name}", so its selection cannot be verified.`); + return; + } + registry.entries.set(identifier, entry); + } + + function failing(name: string, error: unknown): void { + record( + api.it(name, () => { + throw error; + }), + name, + { kind: 'invalid' }, + ); + } + + function desktopIt(name: string, optionsOrBody: DesktopCaseOptions | DesktopSpecBody, maybeBody?: DesktopSpecBody): void { + const hasOptions = typeof optionsOrBody !== 'function'; + const body = hasOptions ? maybeBody : optionsOrBody; + const title = typeof name === 'string' && name.trim() ? name : 'desktopIt'; + const source = `desktopIt("${title}")`; + let descriptor: ResolvedDesktopCaseOptions; + let decision: ReturnType; + try { + if (title !== name) throw new TypeError('desktopIt requires a non-empty name.'); + if (typeof body !== 'function') throw new TypeError(`${source} requires a test function.`); + const declared = validateDesktopCaseOptions(hasOptions ? optionsOrBody : {}, `${source} options`); + descriptor = mergeDesktopCaseOptions(registry.groups.at(-1), declared, source); + decision = registry.selection ? decideDesktopCase(descriptor, registry.selection, source) : { kind: 'run' }; + } catch (error) { + failing(title, error); + return; + } + if (decision.kind === 'run') { + record(api.it(name, body), name, { kind: 'run', descriptor }); + return; + } + // A placeholder keeps the spec out of Jasmine's hooks and body without reporting it as a skipped test. + const entry: DesktopSpecEntry = + decision.kind === 'excluded' ? { kind: 'excluded', reason: decision.reason } : { kind: 'gap', gap: decision.gap }; + record(api.xit(name), name, entry); + } + + function desktopDescribe(name: string, optionsOrBody: DesktopGroupOptions | DesktopGroupBody, maybeBody?: DesktopGroupBody): void { + const hasOptions = typeof optionsOrBody !== 'function'; + const body = hasOptions ? maybeBody : optionsOrBody; + const title = typeof name === 'string' && name.trim() ? name : 'desktopDescribe'; + const source = `desktopDescribe("${title}")`; + let group: ResolvedDesktopCaseOptions; + try { + if (title !== name) throw new TypeError('desktopDescribe requires a non-empty name.'); + if (typeof body !== 'function') throw new TypeError(`${source} requires a body.`); + const declared = validateDesktopGroupOptions(hasOptions ? optionsOrBody : {}, `${source} options`); + group = mergeDesktopCaseOptions(registry.groups.at(-1), declared, source); + } catch (error) { + // The body does not run, so nothing inside an unreadable group is selected by accident. + api.describe(title, () => failing('has invalid desktop options', error)); + return; + } + api.describe(name, function (this: unknown) { + registry.groups.push(group); + try { + body.call(this); + } finally { + registry.groups.pop(); + } + }); + } + + return { desktopIt, desktopDescribe }; +} diff --git a/packages/agentic/desktop-test/src/jasmine/index.ts b/packages/agentic/desktop-test/src/jasmine/index.ts new file mode 100644 index 0000000000..b602b7c452 --- /dev/null +++ b/packages/agentic/desktop-test/src/jasmine/index.ts @@ -0,0 +1,42 @@ +import { createDesktopJasmine } from './adapter.js'; +import type { DesktopGroupBody, DesktopGroupOptions, DesktopJasmineApi, DesktopSpecBody } from './adapter.js'; +import { desktopJasmineRegistry } from './registry.js'; +import type { DesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; + +export type { + DesktopCaseOptions, + DesktopInput, + DesktopPlatforms, + DesktopRequirement, +} from '@fluentui-react-native/desktop-driver/authoring'; +export type { DesktopGroupBody, DesktopGroupOptions, DesktopSpecBody } from './adapter.js'; + +// The Jasmine globals exist only once a framework has installed them, so they are read when a declaration runs. +const globals = () => globalThis as unknown as DesktopJasmineApi; +const jasmineApi: DesktopJasmineApi = { + describe: (name, body) => globals().describe(name, body), + it: (name, body) => globals().it(name, body), + xit: (name) => globals().xit(name), +}; +const adapter = () => createDesktopJasmine(jasmineApi, desktopJasmineRegistry()); + +/** + * Declares a Jasmine test with the input it needs and where it runs. The descriptor decides whether the test is part of a + * run: `requires` implies the input modality, `platforms` and `scope` narrow where it runs, and `unsupported` or + * `quarantine` report a known gap instead of executing. + */ +export function desktopIt(name: string, body: DesktopSpecBody): void; +export function desktopIt(name: string, options: DesktopCaseOptions, body: DesktopSpecBody): void; +export function desktopIt(name: string, optionsOrBody: DesktopCaseOptions | DesktopSpecBody, maybeBody?: DesktopSpecBody): void { + adapter().desktopIt(name, optionsOrBody, maybeBody); +} + +/** + * Declares a Jasmine group whose options apply to every test inside it: requirements add up, platforms and scope intersect, + * and a nested declaration can only narrow its group. + */ +export function desktopDescribe(name: string, body: DesktopGroupBody): void; +export function desktopDescribe(name: string, options: DesktopGroupOptions, body: DesktopGroupBody): void; +export function desktopDescribe(name: string, optionsOrBody: DesktopGroupOptions | DesktopGroupBody, maybeBody?: DesktopGroupBody): void { + adapter().desktopDescribe(name, optionsOrBody, maybeBody); +} diff --git a/packages/agentic/desktop-test/src/jasmine/registry.ts b/packages/agentic/desktop-test/src/jasmine/registry.ts new file mode 100644 index 0000000000..d927d700f1 --- /dev/null +++ b/packages/agentic/desktop-test/src/jasmine/registry.ts @@ -0,0 +1,72 @@ +import type { + DesktopDeclaredGap, + DesktopExclusionReason, + DesktopRequirement, + ResolvedDesktopCaseOptions, +} from '@fluentui-react-native/desktop-driver/authoring'; +import type { SelectionExclusions } from '../reporting/protocol.js'; +import { desktopTestSelectionEnvironment, parseDesktopTestSelection } from './selection.js'; +import type { DesktopTestSelection } from './selection.js'; + +/** What the adapter decided for a spec it registered. */ +export type DesktopSpecEntry = + /** Executes, unless a preflight found that the host cannot meet what it requires. */ + | { kind: 'run'; descriptor: ResolvedDesktopCaseOptions; missing?: readonly DesktopRequirement[] } + /** Registered as a placeholder that never runs and is counted, not reported as a test. */ + | { kind: 'excluded'; reason: DesktopExclusionReason } + /** Registered as a placeholder that is reported as unsupported, with the reason its author gave. */ + | { kind: 'gap'; gap: DesktopDeclaredGap } + /** A failing spec the adapter registered in place of a declaration it could not accept. */ + | { kind: 'invalid' }; + +/** What one worker process learned while its spec files registered their tests. */ +export type DesktopJasmineRegistry = { + /** Absent when no desktop-test run started this process, in which case declarations are only validated. */ + readonly selection: DesktopTestSelection | undefined; + /** Keyed by the identifier WDIO reports for a spec: its description followed by its Jasmine id. */ + readonly entries: Map; + /** The enclosing `desktopDescribe` options while a group body registers, innermost last. */ + readonly groups: ResolvedDesktopCaseOptions[]; + /** Faults that make the selection untrustworthy, such as a spec the adapter could not identify. */ + readonly errors: string[]; + /** Set once every spec was disabled because of a fault, so the reporter shows the fault instead of a wall of skips. */ + blocked: boolean; +}; + +const registryKey = Symbol.for('@fluentui-react-native/desktop-test/jasmine-registry'); + +/** + * The registry of this process. It lives on `globalThis` because the spec files and the reporter share one worker process + * but may load this module through different paths, such as source and its build. + */ +export function desktopJasmineRegistry(environment: NodeJS.ProcessEnv = process.env): DesktopJasmineRegistry { + const holder = globalThis as { [registryKey]?: DesktopJasmineRegistry }; + return (holder[registryKey] ??= createDesktopJasmineRegistry(environment)); +} + +export function createDesktopJasmineRegistry(environment: NodeJS.ProcessEnv = {}): DesktopJasmineRegistry { + const errors: string[] = []; + let selection: DesktopTestSelection | undefined; + const raw = environment[desktopTestSelectionEnvironment]; + if (raw !== undefined) { + try { + selection = parseDesktopTestSelection(raw); + } catch (error) { + errors.push(error instanceof Error ? error.message : String(error)); + } + } + return { selection, entries: new Map(), groups: [], errors, blocked: false }; +} + +/** The identifier WDIO's Jasmine adapter reports for a spec, so the reporter can find what the adapter decided. */ +export function desktopSpecIdentifier(spec: { description?: unknown; id?: unknown } | undefined): string | undefined { + return typeof spec?.description === 'string' && typeof spec.id === 'string' ? `${spec.description}${spec.id}` : undefined; +} + +export function excludedSpecCounts(registry: DesktopJasmineRegistry): SelectionExclusions { + const excluded = { platform: 0, scope: 0, input: 0 }; + for (const entry of registry.entries.values()) { + if (entry.kind === 'excluded') excluded[entry.reason]++; + } + return excluded; +} diff --git a/packages/agentic/desktop-test/src/jasmine/selection.test.ts b/packages/agentic/desktop-test/src/jasmine/selection.test.ts new file mode 100644 index 0000000000..83edda6692 --- /dev/null +++ b/packages/agentic/desktop-test/src/jasmine/selection.test.ts @@ -0,0 +1,50 @@ +import { + desktopTestSelectionEnvironment, + desktopTestSelectionFromEnvironment, + parseDesktopTestSelection, + serializeDesktopTestSelection, +} from './selection.js'; + +describe('desktop test selection', () => { + test('round-trips a lane, the custom scope domains and the target values', () => { + const raw = serializeDesktopTestSelection({ + endpoint: 'macos', + lane: 'accessible', + testScopes: { flavor: ['dev', 'prod'] }, + values: { 'rn.renderer': 'fabric', flavor: 'dev' }, + }); + + const selection = parseDesktopTestSelection(raw); + + expect(selection).toMatchObject({ endpoint: 'macos', lane: 'accessible', values: { 'rn.renderer': 'fabric', flavor: 'dev' } }); + expect(selection.domains.flavor).toEqual(['dev', 'prod']); + expect(selection.domains['rn.renderer']).toEqual(['fabric', 'paper']); + }); + + test('no lane means a run that only applies platform, scope and declared gaps', () => { + const selection = parseDesktopTestSelection(serializeDesktopTestSelection({ endpoint: 'win32', values: {} })); + + expect(selection.lane).toBeUndefined(); + }); + + test('is absent when no run started the process, and malformed values fail loudly rather than being guessed at', () => { + expect(desktopTestSelectionFromEnvironment({})).toBeUndefined(); + expect(desktopTestSelectionFromEnvironment({ [desktopTestSelectionEnvironment]: '{"endpoint":"win32","values":{}}' })).toMatchObject({ + endpoint: 'win32', + }); + expect(() => desktopTestSelectionFromEnvironment({ [desktopTestSelectionEnvironment]: '' })).toThrow('is not valid JSON'); + }); + + test.each([ + ['not json', 'is not valid JSON'], + ['[]', 'must be an object'], + ['null', 'must be an object'], + ['{"endpoint":"linux","values":{}}', 'endpoint must be one of: macos, windows, win32'], + ['{"endpoint":"win32","lane":"fast","values":{}}', 'lane must be one of: accessible, physical'], + ['{"endpoint":"win32","values":{},"extra":1}', 'unknown key "extra"'], + ['{"endpoint":"win32","values":{"flavor":"dev"}}', 'flavor'], + ['{"endpoint":"win32","testScopes":{"rn.renderer":["x"]},"values":{}}', 'rn.renderer'], + ])('rejects %s', (raw, message) => { + expect(() => parseDesktopTestSelection(raw)).toThrow(message); + }); +}); diff --git a/packages/agentic/desktop-test/src/jasmine/selection.ts b/packages/agentic/desktop-test/src/jasmine/selection.ts new file mode 100644 index 0000000000..b7e23d54db --- /dev/null +++ b/packages/agentic/desktop-test/src/jasmine/selection.ts @@ -0,0 +1,59 @@ +import type { DesktopEndpoint } from '@fluentui-react-native/desktop-driver'; +import { desktopInputLanes, resolveDesktopScopeDomains, resolveDesktopScopeValues } from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopCaseTarget, DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; + +/** Carries a run's selection from its launcher to its WDIO workers. The value is data that is parsed, never evaluated. */ +export const desktopTestSelectionEnvironment = 'DESKTOP_TEST_SELECTION'; + +/** What a worker judges each declared case against. */ +export type DesktopTestSelection = Omit; + +/** The serialized selection: the custom scope domains and the target's values, which a worker validates again. */ +export type DesktopTestSelectionRequest = { + endpoint: DesktopEndpoint; + lane?: DesktopInputLane; + testScopes?: unknown; + values: Readonly>; +}; + +const endpoints: readonly string[] = ['macos', 'windows', 'win32']; + +export function serializeDesktopTestSelection(request: DesktopTestSelectionRequest): string { + const { endpoint, lane, testScopes, values } = request; + return JSON.stringify({ endpoint, ...(lane ? { lane } : {}), ...(testScopes ? { testScopes } : {}), values }); +} + +/** Validates the selection a launcher sent. A malformed one is an error, so a worker never guesses what to run. */ +export function parseDesktopTestSelection(raw: string): DesktopTestSelection { + let value: unknown; + try { + value = JSON.parse(raw); + } catch { + throw new TypeError(`${desktopTestSelectionEnvironment} is not valid JSON.`); + } + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new TypeError(`${desktopTestSelectionEnvironment} must be an object.`); + } + const selection = value as Record; + const unknown = Object.keys(selection).find((key) => !['endpoint', 'lane', 'testScopes', 'values'].includes(key)); + if (unknown !== undefined) throw new TypeError(`${desktopTestSelectionEnvironment} has unknown key "${unknown}".`); + if (typeof selection.endpoint !== 'string' || !endpoints.includes(selection.endpoint)) { + throw new TypeError(`${desktopTestSelectionEnvironment}.endpoint must be one of: ${endpoints.join(', ')}.`); + } + if (selection.lane !== undefined && !desktopInputLanes.some((lane) => lane === selection.lane)) { + throw new TypeError(`${desktopTestSelectionEnvironment}.lane must be one of: ${desktopInputLanes.join(', ')}.`); + } + const domains = resolveDesktopScopeDomains(selection.testScopes, `${desktopTestSelectionEnvironment}.testScopes`); + return { + endpoint: selection.endpoint as DesktopEndpoint, + ...(selection.lane ? { lane: selection.lane as DesktopInputLane } : {}), + domains, + values: resolveDesktopScopeValues(selection.values ?? {}, domains, `${desktopTestSelectionEnvironment}.values`), + }; +} + +/** The selection of this process, or undefined when no desktop-test run started it. */ +export function desktopTestSelectionFromEnvironment(environment: NodeJS.ProcessEnv = process.env): DesktopTestSelection | undefined { + const raw = environment[desktopTestSelectionEnvironment]; + return raw === undefined ? undefined : parseDesktopTestSelection(raw); +} diff --git a/packages/agentic/desktop-test/src/jasmine/specSelection.test.ts b/packages/agentic/desktop-test/src/jasmine/specSelection.test.ts new file mode 100644 index 0000000000..85c105ec72 --- /dev/null +++ b/packages/agentic/desktop-test/src/jasmine/specSelection.test.ts @@ -0,0 +1,183 @@ +import { createDesktopJasmine } from './adapter.js'; +import type { DesktopJasmineApi } from './adapter.js'; +import { createDesktopJasmineRegistry } from './registry.js'; +import { desktopTestSelectionEnvironment, serializeDesktopTestSelection } from './selection.js'; +import { selectDesktopSpecs } from './specSelection.js'; +import type { DesktopJasmineEnvironment } from './specSelection.js'; + +type Node = { id?: string; description: string; children?: Node[] }; +type SpecFilter = (spec: unknown) => boolean; + +const masked = { keyboard: false, physicalClick: false, wheel: false }; +const body = () => undefined; + +/** A Jasmine-shaped suite tree that the adapter registers into, and the environment that exposes it. */ +function setup(lane?: 'accessible' | 'physical', configured?: SpecFilter) { + const registry = createDesktopJasmineRegistry({ + [desktopTestSelectionEnvironment]: serializeDesktopTestSelection({ + endpoint: 'win32', + ...(lane ? { lane } : {}), + values: { 'rn.renderer': 'paper' }, + }), + }); + const root: Node = { description: 'top suite', children: [] }; + const stack = [root]; + let next = 0; + const add = (description: string) => { + const spec = { id: `spec${next++}`, description }; + stack.at(-1)!.children!.push(spec); + return spec; + }; + const api: DesktopJasmineApi = { + describe(description, suiteBody) { + const suite: Node = { description, children: [] }; + stack.at(-1)!.children!.push(suite); + stack.push(suite); + try { + suiteBody.call({}); + } finally { + stack.pop(); + } + }, + it: (description) => add(description), + xit: (description) => add(description), + }; + let specFilter = configured; + const configure = jest.fn((configuration: { specFilter: SpecFilter }) => { + specFilter = configuration.specFilter; + }); + const environment: DesktopJasmineEnvironment = { configuration: () => ({ specFilter }), configure, topSuite: () => root }; + const spec = (description: string) => { + const found = root.children!.flatMap((child) => (child.children ?? [child]) as Node[]).find((node) => node.description === description); + if (!found) throw new Error(`No spec named ${description}.`); + return found; + }; + return { + registry, + environment, + configure, + api, + spec, + runs: (description: string) => specFilter?.(spec(description)) ?? true, + ...createDesktopJasmine(api, registry), + }; +} + +describe('selectDesktopSpecs preflight', () => { + test('excludes a spec the host cannot support, so its hooks never run, and leaves the rest', () => { + const { desktopIt, environment, registry, runs } = setup('physical'); + desktopIt('needs keyboard', { requires: ['keyboard'] }, body); + desktopIt('needs click', { requires: ['physical-click'] }, body); + desktopIt('needs nothing', body); + + selectDesktopSpecs(registry, environment, { keyboard: false, physicalClick: true }); + + expect([runs('needs keyboard'), runs('needs click'), runs('needs nothing')]).toEqual([false, true, true]); + expect(registry.entries.get('needs keyboardspec0')).toMatchObject({ kind: 'run', missing: ['keyboard'] }); + expect(registry.entries.get('needs clickspec1')).not.toHaveProperty('missing'); + }); + + test('keeps the filter the user already configured', () => { + const { desktopIt, environment, registry, runs } = setup('physical', (spec) => (spec as Node).description !== 'user filtered'); + desktopIt('user filtered', body); + desktopIt('needs keyboard', { requires: ['keyboard'] }, body); + desktopIt('kept', body); + + selectDesktopSpecs(registry, environment, { keyboard: false }); + + expect([runs('user filtered'), runs('needs keyboard'), runs('kept')]).toEqual([false, false, true]); + }); + + test('does not touch the filter when the host meets every requirement', () => { + const { configure, desktopIt, environment, registry } = setup('physical'); + desktopIt('needs keyboard', { requires: ['keyboard'] }, body); + + selectDesktopSpecs(registry, environment, { keyboard: true }); + + expect(configure).not.toHaveBeenCalled(); + }); + + test('judges a click requirement by what the lane clicks with', () => { + const physical = setup('physical'); + physical.desktopIt('clicks', { requires: ['click'] }, body); + selectDesktopSpecs(physical.registry, physical.environment, { physicalClick: false, accessibilityClick: true }); + expect(physical.runs('clicks')).toBe(false); + + const accessible = setup('accessible'); + accessible.desktopIt('clicks', { requires: ['click'] }, body); + selectDesktopSpecs(accessible.registry, accessible.environment, { ...masked, accessibilityClick: true }); + expect(accessible.runs('clicks')).toBe(true); + }); +}); + +describe('selectDesktopSpecs fail-closed checks', () => { + test('an accessible lane on a host that still offers physical input disables every spec and says why', () => { + const { desktopIt, environment, registry, runs } = setup('accessible'); + desktopIt('fine', body); + + expect(() => selectDesktopSpecs(registry, environment, { keyboard: true, physicalClick: false })).toThrow( + 'The accessible input lane needs a host that reports keyboard, wheel disabled, and this one does not.', + ); + expect(registry.blocked).toBe(true); + expect(runs('fine')).toBe(false); + }); + + test('an absent feature report counts as unmasked', () => { + const { environment, registry } = setup('accessible'); + + expect(() => selectDesktopSpecs(registry, environment, undefined)).toThrow(/keyboard, physicalClick, wheel disabled/); + }); + + test('an accessible lane on a masked host runs', () => { + const { desktopIt, environment, registry, runs } = setup('accessible'); + desktopIt('fine', body); + + selectDesktopSpecs(registry, environment, masked); + + expect(registry.blocked).toBe(false); + expect(runs('fine')).toBe(true); + }); + + test('the physical lane needs no mask', () => { + const { desktopIt, environment, registry } = setup('physical'); + desktopIt('fine', body); + + expect(() => selectDesktopSpecs(registry, environment, { keyboard: true })).not.toThrow(); + }); + + test('a lane refuses specs that were not declared with desktopIt, however many', () => { + const { api, desktopIt, environment, registry, runs } = setup('physical'); + desktopIt('declared', body); + api.describe('legacy', () => { + for (let index = 0; index < 12; index++) api.it(`spec ${index}`, body); + }); + + expect(() => selectDesktopSpecs(registry, environment, {})).toThrow( + /12 specs were not declared with desktopIt or desktopDescribe: "legacy spec 0", .*"legacy spec 9" and 2 more\./, + ); + expect(registry.blocked).toBe(true); + expect(runs('declared')).toBe(false); + }); + + test('a lane cannot be verified without Jasmine, so it blocks', () => { + const { registry } = setup('physical'); + + expect(() => selectDesktopSpecs(registry, undefined, {})).toThrow('Jasmine did not expose its environment'); + expect(registry.blocked).toBe(true); + }); + + test('without a lane, undeclared specs are tolerated', () => { + const { api, environment, registry } = setup(); + api.it('legacy', body); + + expect(() => selectDesktopSpecs(registry, environment, {})).not.toThrow(); + }); + + test('a fault found while registering blocks the run even without a lane, together with any other problem', () => { + const { environment, registry } = setup('accessible'); + registry.errors.push('Jasmine did not identify the spec registered for "ghost", so its selection cannot be verified.'); + + expect(() => selectDesktopSpecs(registry, environment, {})).toThrow(/ghost[\s\S]*accessible input lane/); + expect(registry.blocked).toBe(true); + }); +}); diff --git a/packages/agentic/desktop-test/src/jasmine/specSelection.ts b/packages/agentic/desktop-test/src/jasmine/specSelection.ts new file mode 100644 index 0000000000..ac3c059544 --- /dev/null +++ b/packages/agentic/desktop-test/src/jasmine/specSelection.ts @@ -0,0 +1,96 @@ +import { + missingDesktopRequirements, + resolveDesktopClickMode, + unmaskedAccessibleFeatures, +} from '@fluentui-react-native/desktop-driver/authoring'; +import { desktopSpecIdentifier } from './registry.js'; +import type { DesktopJasmineRegistry } from './registry.js'; + +type JasmineNode = { id?: unknown; description?: unknown; children?: readonly JasmineNode[] }; +type JasmineSpecFilter = (spec: unknown) => boolean; + +/** The part of Jasmine's environment that applying a selection needs. */ +export type DesktopJasmineEnvironment = { + configuration(): { specFilter?: JasmineSpecFilter }; + configure(configuration: { specFilter: JasmineSpecFilter }): void; + topSuite(): JasmineNode; +}; + +/** The environment of the Jasmine this process runs under, if a framework installed one. */ +export function currentDesktopJasmineEnvironment(): DesktopJasmineEnvironment | undefined { + const jasmine = (globalThis as { jasmine?: { getEnv?: () => DesktopJasmineEnvironment } }).jasmine; + return jasmine?.getEnv?.(); +} + +const listedSpecs = 10; + +/** + * Applies a run's selection once the spec files have registered their tests and the host is known, before any test runs. + * + * A fault that makes the selection untrustworthy disables every spec and throws, so a lane never executes a test it cannot + * vouch for. Otherwise, a test whose requirements the host cannot meet is excluded here and reported as unsupported. + */ +export function selectDesktopSpecs( + registry: DesktopJasmineRegistry, + environment: DesktopJasmineEnvironment | undefined, + features: unknown, +): void { + const lane = registry.selection?.lane; + const problems = [...registry.errors]; + if (lane === 'accessible') { + const unmasked = unmaskedAccessibleFeatures(features); + if (unmasked.length > 0) { + problems.push(`The accessible input lane needs a host that reports ${unmasked.join(', ')} disabled, and this one does not.`); + } + } + if (lane) { + const undeclared = environment ? undeclaredSpecs(registry, environment) : undefined; + if (!undeclared) problems.push('Jasmine did not expose its environment, so the specs of an input lane cannot be verified.'); + else if (undeclared.length > 0) { + const listed = undeclared.slice(0, listedSpecs).join(', '); + const more = undeclared.length > listedSpecs ? ` and ${undeclared.length - listedSpecs} more` : ''; + problems.push( + `An input lane runs only declared tests, but ${undeclared.length} ${undeclared.length === 1 ? 'spec was' : 'specs were'} not declared with desktopIt or desktopDescribe: ${listed}${more}.`, + ); + } + } + if (problems.length > 0) { + registry.blocked = true; + environment?.configure({ specFilter: () => false }); + throw new Error(problems.join('\n')); + } + if (!environment) return; + // The session's click mode is the lane's, so this is what a `click` requirement would really do. + const clickMode = resolveDesktopClickMode(undefined, 'auto', lane); + const unavailable = new Set(); + for (const [identifier, entry] of registry.entries) { + if (entry.kind !== 'run') continue; + const missing = missingDesktopRequirements(entry.descriptor, features as Readonly> | undefined, clickMode); + if (missing.length === 0) continue; + entry.missing = missing; + unavailable.add(identifier); + } + if (unavailable.size === 0) return; + const configured = environment.configuration().specFilter; + // A filtered spec is excluded before Jasmine runs its hooks, which an executed-then-skipped spec could not avoid. + environment.configure({ + specFilter: (spec) => { + const identifier = desktopSpecIdentifier(spec as { description?: unknown; id?: unknown }); + return !(identifier !== undefined && unavailable.has(identifier)) && (configured?.(spec) ?? true); + }, + }); +} + +/** The full names of the specs no `desktopIt` registered. */ +function undeclaredSpecs(registry: DesktopJasmineRegistry, environment: DesktopJasmineEnvironment): string[] { + const undeclared: string[] = []; + const visit = (node: JasmineNode, path: readonly string[]) => { + for (const child of node.children ?? []) { + const names = [...path, String(child.description)]; + if (child.children) visit(child, names); + else if (!registry.entries.has(desktopSpecIdentifier(child) ?? '')) undeclared.push(`"${names.join(' ')}"`); + } + }; + visit(environment.topSuite(), []); + return undeclared; +} diff --git a/packages/agentic/desktop-test/src/reporting/collector.test.ts b/packages/agentic/desktop-test/src/reporting/collector.test.ts index 40d556cf07..3d2d3f3f39 100644 --- a/packages/agentic/desktop-test/src/reporting/collector.test.ts +++ b/packages/agentic/desktop-test/src/reporting/collector.test.ts @@ -3,6 +3,127 @@ import os from 'node:os'; import path from 'node:path'; import { DesktopRunReporter } from './collector.js'; +import type { DesktopInputLaneReport } from './results.js'; + +describe('desktop run reporter input lanes', () => { + const accessible: DesktopInputLaneReport = { input: 'accessible', source: 'option', masking: 'host-masked', policyEnforced: false }; + let artifactsRoot: string; + + beforeEach(() => { + artifactsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-desktop-lane-')); + }); + + afterEach(() => { + fs.rmSync(artifactsRoot, { force: true, recursive: true }); + }); + + function start(reporterMode: 'plain' | 'ndjson', lane?: DesktopInputLaneReport) { + let written = ''; + const output = { + write: (chunk: string) => { + written += chunk; + return true; + }, + }; + const reporter = new DesktopRunReporter({ + artifactsRoot, + errorOutput: output, + output, + reporterMode, + summary: false, + ...(lane ? { lane } : {}), + }); + return { reporter, output: () => written }; + } + + function unsupported(reporter: DesktopRunReporter, testId: string, gap?: 'unsupported' | 'quarantine'): void { + reporter.emit({ + kind: 'test-result', + payload: { + testId, + title: testId, + attempt: 0, + status: 'unsupported', + durationMs: 0, + reason: 'Not available.', + ...(gap ? { gap } : {}), + }, + }); + } + + function passed(reporter: DesktopRunReporter, testId: string): void { + reporter.emit({ kind: 'test-result', payload: { testId, title: testId, attempt: 0, status: 'passed', durationMs: 1 } }); + } + + test('persists the lane and sums exclusions from every producer without selecting them as tests', async () => { + const { reporter } = start('plain', accessible); + reporter.emit({ kind: 'selection', payload: { excluded: { platform: 2, scope: 0, input: 5 } } }); + reporter.emit({ kind: 'selection', payload: { excluded: { platform: 1, scope: 1, input: 4 } } }); + passed(reporter, 'case'); + const result = await reporter.finish(); + + expect(result).toMatchObject({ + status: 'passed', + lane: accessible, + selection: { excluded: { platform: 3, scope: 1, input: 9 } }, + counts: { selected: 1, passed: 1 }, + }); + expect(JSON.parse(fs.readFileSync(path.join(reporter.root, 'run.json'), 'utf8'))).toMatchObject({ + lane: accessible, + selection: { excluded: { platform: 3, scope: 1, input: 9 } }, + }); + }); + + test('counts a keyed selection once, so a retried worker does not double its exclusions', async () => { + const { reporter } = start('plain', accessible); + const excluded = { platform: 1, scope: 2, input: 3 }; + reporter.emit({ kind: 'selection', payload: { excluded, key: 'wdio-group' } }); + reporter.emit({ kind: 'selection', payload: { excluded, key: 'wdio-group' } }); + reporter.emit({ kind: 'selection', payload: { excluded: { platform: 0, scope: 0, input: 1 }, key: 'another-group' } }); + passed(reporter, 'case'); + + expect(await reporter.finish()).toMatchObject({ selection: { excluded: { platform: 1, scope: 2, input: 4 } } }); + }); + + test('omits the selection when no producer reported one', async () => { + const { reporter } = start('plain', accessible); + passed(reporter, 'case'); + + expect(await reporter.finish()).not.toHaveProperty('selection'); + }); + + test('a lane run is incomplete when a case was unsupported without a declaration', async () => { + const { reporter, output } = start('plain', accessible); + passed(reporter, 'case'); + unsupported(reporter, 'declared', 'unsupported'); + unsupported(reporter, 'missing'); + reporter.emit({ kind: 'selection', payload: { excluded: { platform: 0, scope: 0, input: 3 } } }); + const result = await reporter.finish(); + + expect(result.status).toBe('incomplete'); + expect(output()).toContain('2 unsupported (1 declared, 0 quarantined)'); + expect(output()).toContain('[desktop] input accessible (host-masked) from --input; excluded 0 by platform, 0 by scope, 3 by input'); + }); + + test('the same unsupported case does not fail a compatibility run', async () => { + const { reporter, output } = start('plain'); + passed(reporter, 'case'); + unsupported(reporter, 'missing'); + const result = await reporter.finish(); + + expect(result.status).toBe('passed'); + expect(result).not.toHaveProperty('lane'); + expect(output()).not.toContain('excluded'); + }); + + test('the ndjson start message names the lane', async () => { + const { reporter, output } = start('ndjson', accessible); + passed(reporter, 'case'); + await reporter.finish(); + + expect(JSON.parse(output().split('\n')[0])).toMatchObject({ type: 'start', lane: accessible }); + }); +}); describe('desktop run reporter final output', () => { let artifactsRoot: string; diff --git a/packages/agentic/desktop-test/src/reporting/collector.ts b/packages/agentic/desktop-test/src/reporting/collector.ts index 4acbd00b8d..c2d81db4cf 100644 --- a/packages/agentic/desktop-test/src/reporting/collector.ts +++ b/packages/agentic/desktop-test/src/reporting/collector.ts @@ -9,12 +9,20 @@ import { DesktopReportedError, redactDesktopSecrets, sanitizeDesktopReport, + selectionExclusionReasons, serializeReportError, validateDesktopEvent, } from './protocol.js'; import type { CollectedDesktopEvent, EventContext, EventEmitter, FailureKind, ReportError, ReportFailure } from './protocol.js'; -import { projectDesktopRunSummary, renderJUnit, renderRunSummary, summarizeDesktopRun } from './results.js'; -import type { DesktopRunResult, DesktopTestResult, DesktopWorkResult } from './results.js'; +import { + describeDesktopSelection, + describeDesktopUnsupported, + projectDesktopRunSummary, + renderJUnit, + renderRunSummary, + summarizeDesktopRun, +} from './results.js'; +import type { DesktopInputLaneReport, DesktopRunResult, DesktopTestResult, DesktopWorkResult } from './results.js'; import { includeDesktopEvent, projectDesktopEvent } from './structured.js'; import type { DesktopEventLevel } from './structured.js'; import type { DesktopReportingConfig } from '../config.js'; @@ -25,6 +33,8 @@ export type ReportOutput = Pick; export type DesktopRunReporterOptions = DesktopReportingConfig & { artifactsRoot: string; context?: EventContext; + /** The input lane this run executes. Lane runs hold every selected case to its declared outcome. */ + lane?: DesktopInputLaneReport; kind?: 'tests' | 'lifecycle'; optional?: boolean; output?: ReportOutput; @@ -68,6 +78,9 @@ export class DesktopRunReporter { private readonly presentedFailures = new Set(); private readonly secrets = new Set(); private readonly artifacts = new Map(); + private readonly excluded = { platform: 0, scope: 0, input: 0 }; + private readonly selectionKeys = new Set(); + private excludedReported = false; private readonly options: DesktopRunReporterOptions; private readonly actions: boolean; private readonly output: ReportOutput; @@ -136,6 +149,7 @@ export class DesktopRunReporter { runId: this.runId, kind: options.kind ?? 'tests', ...(options.context ? { context: options.context } : {}), + ...(options.lane ? { lane: options.lane } : {}), artifactsRoot: clipStructuredText(this.root, 512), }; this.enqueue(() => this.write(this.output, `${JSON.stringify(start)}\n`), 0); @@ -314,6 +328,14 @@ export class DesktopRunReporter { test.status = test.attempts[test.attempts.length - 1].status; test.flaky = test.status === 'passed' && test.attempts.some(({ status }) => status === 'failed'); this.tests.set(p.testId, test); + } else if (event.kind === 'selection') { + this.excludedReported = true; + // A keyed event is idempotent: a retried worker announces the same exclusions again. + if (event.payload.key !== undefined) { + if (this.selectionKeys.has(event.payload.key)) return; + this.selectionKeys.add(event.payload.key); + } + for (const reason of selectionExclusionReasons) this.excluded[reason] += event.payload.excluded[reason]; } else if (event.kind === 'artifact') this.artifacts.set(event.payload.path, event.payload.description); } @@ -495,6 +517,8 @@ export class DesktopRunReporter { finishedAt: new Date().toISOString(), durationMs: Date.now() - this.startedAt.getTime(), context: this.options.context, + ...(this.options.lane ? { lane: this.options.lane } : {}), + ...(this.excludedReported ? { selection: { excluded: { ...this.excluded } } } : {}), optional: this.options.optional, incomplete: [...this.scopes.values()].some(({ finished }) => !finished) || @@ -551,8 +575,10 @@ export class DesktopRunReporter { let result = persist(); const c = result.counts; this.logger.log( - `[desktop] ${result.status.toUpperCase()}: ${c.selected} selected, ${c.executed} executed, ${c.passed} passed, ${c.failed} failed, ${c.skipped} skipped, ${c.unsupported} unsupported, ${c.incomplete} incomplete, ${c.retries} retries, ${c.flaky} flaky; ${result.work.filter(({ status }) => status === 'not-run').length} groups not run; ${path.join(this.root, 'run.json')}`, + `[desktop] ${result.status.toUpperCase()}: ${c.selected} selected, ${c.executed} executed, ${c.passed} passed, ${c.failed} failed, ${c.skipped} skipped, ${describeDesktopUnsupported(c)}, ${c.incomplete} incomplete, ${c.retries} retries, ${c.flaky} flaky; ${result.work.filter(({ status }) => status === 'not-run').length} groups not run; ${path.join(this.root, 'run.json')}`, ); + const selection = describeDesktopSelection(result); + if (selection) this.logger.log(`[desktop] ${selection}`); await this.queue; result = persist(); if (this.presentationFailed && !this.structured) { diff --git a/packages/agentic/desktop-test/src/reporting/index.ts b/packages/agentic/desktop-test/src/reporting/index.ts index 428250bac3..16383d62c2 100644 --- a/packages/agentic/desktop-test/src/reporting/index.ts +++ b/packages/agentic/desktop-test/src/reporting/index.ts @@ -11,11 +11,33 @@ export type { FailureKind, ReportError, ReportFailure, + SelectionExclusions, + TestGap, TestStatus, } from './protocol.js'; export type { DesktopEventRecord } from './structured.js'; export type { DesktopEventLevel } from '@fluentui-react-native/desktop-driver/cli/reporter'; -export { desktopRunExitCode, projectDesktopRunSummary, renderJUnit, renderRunSummary, summarizeDesktopRun } from './results.js'; -export type { DesktopRunResult, DesktopRunSummary, DesktopTestAttempt, DesktopTestResult } from './results.js'; +export { createDesktopInputLane, desktopInputEnvironment, parseDesktopInputLane, requestedDesktopInputLane } from './lane.js'; +export type { DesktopInputLaneName, RequestedDesktopInputLane } from './lane.js'; +export { + describeDesktopInputLane, + describeDesktopSelection, + describeDesktopUnsupported, + desktopInputLaneSources, + desktopRunExitCode, + projectDesktopRunSummary, + renderJUnit, + renderRunSummary, + summarizeDesktopRun, +} from './results.js'; +export type { + DesktopInputLaneReport, + DesktopInputLaneSource, + DesktopRunResult, + DesktopRunSummary, + DesktopSelectionReport, + DesktopTestAttempt, + DesktopTestResult, +} from './results.js'; export { connectReportProducer, DESKTOP_REPORT_CONTEXT, startReportTransport } from './transport.js'; export type { ReportProducer, ReportTransport, ReportTransportContext } from './transport.js'; diff --git a/packages/agentic/desktop-test/src/reporting/lane.test.ts b/packages/agentic/desktop-test/src/reporting/lane.test.ts new file mode 100644 index 0000000000..f10c320dd9 --- /dev/null +++ b/packages/agentic/desktop-test/src/reporting/lane.test.ts @@ -0,0 +1,34 @@ +import { createDesktopInputLane, desktopInputEnvironment, parseDesktopInputLane, requestedDesktopInputLane } from './lane.js'; + +describe('requested desktop input lane', () => { + test('the flag wins over the environment', () => { + expect(requestedDesktopInputLane({ flag: 'physical', environment: 'accessible' })).toEqual({ input: 'physical', source: 'option' }); + }); + + test('the environment applies when there is no flag', () => { + expect(requestedDesktopInputLane({ environment: 'accessible' })).toEqual({ input: 'accessible', source: 'environment' }); + expect(requestedDesktopInputLane({})).toBeUndefined(); + }); + + test('an empty value is invalid rather than unset, and an error names where it came from', () => { + expect(() => requestedDesktopInputLane({ environment: '' })).toThrow(`${desktopInputEnvironment} must be accessible or physical.`); + expect(() => requestedDesktopInputLane({ flag: 'fast' })).toThrow('--input must be accessible or physical.'); + expect(() => parseDesktopInputLane('Physical', 'x')).toThrow(expect.objectContaining({ code: 'config' })); + }); +}); + +describe('createDesktopInputLane', () => { + test('an accessible lane says how it was masked, and neither lane claims enforcement yet', () => { + expect(createDesktopInputLane('accessible', 'option')).toEqual({ + input: 'accessible', + source: 'option', + masking: 'host-masked', + policyEnforced: false, + }); + expect(createDesktopInputLane('accessible', 'driver', 'verified-masked')).toMatchObject({ + masking: 'verified-masked', + source: 'driver', + }); + expect(createDesktopInputLane('physical', 'environment')).toEqual({ input: 'physical', source: 'environment', policyEnforced: false }); + }); +}); diff --git a/packages/agentic/desktop-test/src/reporting/lane.ts b/packages/agentic/desktop-test/src/reporting/lane.ts new file mode 100644 index 0000000000..49064f01e2 --- /dev/null +++ b/packages/agentic/desktop-test/src/reporting/lane.ts @@ -0,0 +1,37 @@ +import type { DesktopInputLaneReport, DesktopInputLaneSource } from './results.js'; + +/** Selects the input lane of a `test` run when `--input` is absent. */ +export const desktopInputEnvironment = 'DESKTOP_INPUT'; + +export type DesktopInputLaneName = DesktopInputLaneReport['input']; + +export type RequestedDesktopInputLane = { input: DesktopInputLaneName; source: Extract }; + +/** Validates a requested lane. `source` names where the value came from, so a bad one points at the right place. */ +export function parseDesktopInputLane(value: string, source: string): DesktopInputLaneName { + if (value !== 'accessible' && value !== 'physical') { + throw Object.assign(new TypeError(`${source} must be accessible or physical.`), { code: 'config' }); + } + return value; +} + +/** The lane a command was asked for: `--input`, then `DESKTOP_INPUT`. An empty environment value is invalid, not unset. */ +export function requestedDesktopInputLane(request: { flag?: string; environment?: string }): RequestedDesktopInputLane | undefined { + if (request.flag !== undefined) return { input: parseDesktopInputLane(request.flag, '--input'), source: 'option' }; + if (request.environment !== undefined) { + return { input: parseDesktopInputLane(request.environment, desktopInputEnvironment), source: 'environment' }; + } + return undefined; +} + +/** + * How an accessible lane was masked. `host-masked` means this command started the host with keyboard, physical click and + * wheel disabled. `verified-masked` means it attached to a host and checked the mask before running anything. + */ +export function createDesktopInputLane( + input: DesktopInputLaneName, + source: DesktopInputLaneSource, + masking: 'host-masked' | 'verified-masked' = 'host-masked', +): DesktopInputLaneReport { + return input === 'physical' ? { input, source, policyEnforced: false } : { input, source, masking, policyEnforced: false }; +} diff --git a/packages/agentic/desktop-test/src/reporting/protocol.test.ts b/packages/agentic/desktop-test/src/reporting/protocol.test.ts index 19f5fc1fac..2462042655 100644 --- a/packages/agentic/desktop-test/src/reporting/protocol.test.ts +++ b/packages/agentic/desktop-test/src/reporting/protocol.test.ts @@ -1,4 +1,52 @@ -import { redactDesktopSecrets } from './protocol'; +import { redactDesktopSecrets, validateDesktopEvent } from './protocol'; + +describe('declared gaps and selection events', () => { + function event(kind: string, payload: Record): unknown { + return { + schemaVersion: 1, + runId: 'run-id', + eventId: 'event-id', + sourceId: 'worker', + sourceSequence: 1, + scopeId: 'scope-id', + timestamp: new Date(0).toISOString(), + kind, + payload, + }; + } + const result = { testId: 'test', title: 'test', attempt: 0, durationMs: 0, reason: 'Declared in the story.' }; + + test.each(['unsupported', 'quarantine'])('accepts a %s gap on an unsupported result', (gap) => { + expect(() => validateDesktopEvent(event('test-result', { ...result, status: 'unsupported', gap }))).not.toThrow(); + }); + + test.each(['skipped', 'failed', 'passed'])('rejects a declared gap on a %s result', (status) => { + expect(() => validateDesktopEvent(event('test-result', { ...result, status, gap: 'unsupported' }))).toThrow(); + }); + + test('rejects an unknown gap kind', () => { + expect(() => validateDesktopEvent(event('test-result', { ...result, status: 'unsupported', gap: 'flaky' }))).toThrow(); + }); + + test('accepts exactly the three non-negative integer exclusion counts', () => { + expect(() => validateDesktopEvent(event('selection', { excluded: { platform: 1, scope: 0, input: 12 } }))).not.toThrow(); + }); + + test('accepts an optional idempotency key and rejects an empty one', () => { + const excluded = { platform: 1, scope: 0, input: 0 }; + expect(() => validateDesktopEvent(event('selection', { excluded, key: 'wdio-group' }))).not.toThrow(); + expect(() => validateDesktopEvent(event('selection', { excluded, key: '' }))).toThrow(); + }); + + test.each([ + ['a missing reason', { platform: 1, scope: 0 }], + ['an unknown reason', { platform: 1, scope: 0, input: 0, flaky: 1 }], + ['a negative count', { platform: -1, scope: 0, input: 0 }], + ['a fractional count', { platform: 0.5, scope: 0, input: 0 }], + ])('rejects a selection with %s', (_name, excluded) => { + expect(() => validateDesktopEvent(event('selection', { excluded }))).toThrow(); + }); +}); describe('redactDesktopSecrets', () => { test('redacts known secrets, longest first', () => { diff --git a/packages/agentic/desktop-test/src/reporting/protocol.ts b/packages/agentic/desktop-test/src/reporting/protocol.ts index 447455b715..524c10f50b 100644 --- a/packages/agentic/desktop-test/src/reporting/protocol.ts +++ b/packages/agentic/desktop-test/src/reporting/protocol.ts @@ -15,6 +15,12 @@ export class DesktopReportedError extends Error { } } export type TestStatus = 'passed' | 'failed' | 'skipped' | 'unsupported' | 'cancelled' | 'not-run' | 'incomplete'; +/** A declared gap: the producer knows the case cannot run here and says why, so it is not a silent skip. */ +export type TestGap = 'unsupported' | 'quarantine'; +export const testGaps: readonly TestGap[] = ['unsupported', 'quarantine']; +/** Cases a producer removed from its selection before any session existed, by the reason they were removed. */ +export type SelectionExclusions = { platform: number; scope: number; input: number }; +export const selectionExclusionReasons: readonly (keyof SelectionExclusions)[] = ['platform', 'scope', 'input']; export type EventContext = { endpoint?: 'macos' | 'windows' | 'win32'; renderer?: 'fabric' | 'paper'; @@ -43,9 +49,11 @@ export type EventData = status: TestStatus; durationMs: number; reason?: string; + gap?: TestGap; failure?: ReportFailure; }; } + | { kind: 'selection'; payload: { excluded: SelectionExclusions; key?: string } } | { kind: 'artifact'; payload: { path: string; description: string } } | { kind: 'producer-finished'; payload: { status: 'passed' | 'failed' } } | { kind: 'run-finished'; payload: { status: TestStatus; resultPath: string } }; @@ -74,7 +82,8 @@ const payloadFields = { diagnostic: ['level', 'message', 'failure'], output: ['stream', 'text'], 'test-start': ['testId', 'title', 'attempt'], - 'test-result': ['testId', 'title', 'attempt', 'status', 'durationMs', 'reason', 'failure'], + 'test-result': ['testId', 'title', 'attempt', 'status', 'durationMs', 'reason', 'gap', 'failure'], + selection: ['excluded', 'key'], artifact: ['path', 'description'], 'producer-finished': ['status'], 'run-finished': ['status', 'resultPath'], @@ -178,9 +187,20 @@ export function validateDesktopEvent(value: unknown): asserts value is DesktopEv throw new TypeError('Non-executed test results require a reason.'); } if (payload.status === 'failed' && !payload.failure) throw new TypeError('Failed test results require their original failure.'); + if (payload.gap !== undefined) { + member(payload.gap, testGaps); + if (payload.status !== 'unsupported') throw new TypeError('A declared gap is only valid on an unsupported test result.'); + } if (payload.failure !== undefined) validateFailure(payload.failure); } break; + case 'selection': { + const excluded = record(payload.excluded, 'excluded'); + exactKeys(excluded, selectionExclusionReasons); + for (const reason of selectionExclusionReasons) integer(excluded[reason], `excluded.${reason}`, 0); + if (payload.key !== undefined) requireText(payload.key, 'selection key'); + break; + } case 'artifact': requireText(payload.path, 'artifact path'); requireText(payload.description, 'artifact description'); diff --git a/packages/agentic/desktop-test/src/reporting/results.test.ts b/packages/agentic/desktop-test/src/reporting/results.test.ts index af29cd80c3..82208acd89 100644 --- a/packages/agentic/desktop-test/src/reporting/results.test.ts +++ b/packages/agentic/desktop-test/src/reporting/results.test.ts @@ -1,7 +1,174 @@ import os from 'node:os'; import path from 'node:path'; -import { projectDesktopRunSummary, summarizeDesktopRun } from './results.js'; +import { + describeDesktopSelection, + describeDesktopUnsupported, + projectDesktopRunSummary, + renderJUnit, + renderRunSummary, + summarizeDesktopRun, +} from './results.js'; +import type { DesktopInputLaneReport, DesktopRunResult, DesktopTestResult } from './results.js'; + +const accessible: DesktopInputLaneReport = { input: 'accessible', source: 'option', masking: 'host-masked', policyEnforced: false }; + +function outcome( + status: DesktopTestResult['status'], + gap?: 'unsupported' | 'quarantine', + testId = `${status}-${gap ?? 'plain'}`, +): DesktopTestResult { + return { + testId, + title: testId, + status, + flaky: false, + attempts: [ + { + attempt: 0, + status, + durationMs: 1, + ...(status === 'passed' ? {} : { reason: 'Not executed.' }), + ...(gap ? { gap } : {}), + }, + ], + }; +} + +function summarize( + tests: DesktopTestResult[], + extra: { lane?: DesktopInputLaneReport; optional?: boolean; kind?: 'tests' | 'lifecycle' } = {}, +): DesktopRunResult { + const { lane, optional, kind = 'tests' } = extra; + return summarizeDesktopRun({ + schemaVersion: 1, + runId: 'run-id', + kind, + startedAt: new Date(0).toISOString(), + finishedAt: new Date(1).toISOString(), + durationMs: 1, + ...(lane ? { lane } : {}), + ...(optional !== undefined ? { optional } : {}), + tests, + work: [], + failures: [], + artifacts: [], + }); +} + +describe('input lane verdicts', () => { + test('compatibility runs keep skipped and unsupported cases non-blocking', () => { + expect(summarize([outcome('passed'), outcome('skipped'), outcome('unsupported')]).status).toBe('passed'); + }); + + test.each([ + ['unsupported', 'declaredUnsupported'], + ['quarantine', 'quarantined'], + ] as const)('a declared %s gap is a deliberate gap, not a lane failure', (gap, subset) => { + const result = summarize([outcome('passed'), outcome('unsupported', gap)], { lane: accessible }); + + expect(result.status).toBe('passed'); + expect(result.counts).toMatchObject({ + unsupported: 1, + declaredUnsupported: Number(gap === 'unsupported'), + quarantined: Number(gap === 'quarantine'), + }); + expect(result.counts[subset]).toBe(1); + }); + + test('an undeclared unsupported case means the lane did not cover its selection', () => { + const result = summarize([outcome('passed'), outcome('unsupported')], { lane: accessible }); + + expect(result.status).toBe('incomplete'); + expect(result.counts).toMatchObject({ unsupported: 1, declaredUnsupported: 0, quarantined: 0 }); + }); + + test('a skipped case means the lane did not cover its selection', () => { + expect(summarize([outcome('passed'), outcome('skipped')], { lane: accessible }).status).toBe('incomplete'); + }); + + test('optional never masks a lane that skipped or executed nothing', () => { + expect(summarize([outcome('passed'), outcome('skipped')], { lane: accessible, optional: true }).status).toBe('incomplete'); + expect(summarize([outcome('unsupported', 'unsupported')], { lane: accessible, optional: true }).status).toBe('incomplete'); + expect(summarize([outcome('skipped')], { optional: true }).status).toBe('skipped'); + }); + + test('an empty lane selection is incomplete', () => { + expect(summarize([], { lane: accessible }).status).toBe('incomplete'); + }); + + test('failures still fail a lane run', () => { + const failed = outcome('failed'); + failed.attempts[0].failure = { failureId: 'f', kind: 'test', error: { name: 'Error', message: 'boom' } }; + + expect(summarize([outcome('passed'), failed], { lane: accessible }).status).toBe('failed'); + }); + + test('lifecycle runs ignore the lane', () => { + expect(summarize([], { lane: accessible, kind: 'lifecycle' }).status).toBe('passed'); + }); +}); + +describe('input lane and exclusion reporting', () => { + const selection = { excluded: { platform: 2, scope: 1, input: 7 } }; + const result = (): DesktopRunResult => ({ + ...summarize([outcome('passed'), outcome('unsupported', 'unsupported'), outcome('unsupported', 'quarantine', 'quarantined-case')], { + lane: accessible, + }), + selection, + }); + + test('names the lane, its source and each exclusion on one line', () => { + expect(describeDesktopSelection(result())).toBe( + 'input accessible (host-masked) from --input; excluded 2 by platform, 1 by scope, 7 by input', + ); + expect(describeDesktopSelection({ lane: { input: 'physical', source: 'environment', policyEnforced: false } })).toBe( + 'input physical pilot from DESKTOP_INPUT', + ); + expect( + describeDesktopSelection({ lane: { input: 'accessible', source: 'driver', masking: 'verified-masked', policyEnforced: false } }), + ).toBe('input accessible (verified-masked) from the driver instance'); + expect(describeDesktopSelection({})).toBeUndefined(); + }); + + test('breaks unsupported cases into declared and quarantined', () => { + expect(describeDesktopUnsupported(result().counts)).toBe('2 unsupported (1 declared, 1 quarantined)'); + }); + + test('JUnit records the lane and exclusions and labels declared gaps', () => { + const xml = renderJUnit(result()); + + expect(xml).toContain(''); + expect(xml).toContain(''); + expect(xml).toContain(''); + expect(xml).toContain(''); + expect(xml).toContain(''); + expect(xml).toContain(''); + expect(xml).toContain(''); + expect(xml).toContain('skipped="2"'); + }); + + test('JUnit stays free of lane properties for compatibility runs', () => { + expect(renderJUnit(summarize([outcome('passed')]))).not.toContain(''); + }); + + test('the Markdown summary leads with the selection and splits unsupported counts', () => { + const markdown = renderRunSummary(result()); + + expect(markdown).toContain('Selection: input accessible (host-masked) from --input; excluded 2 by platform, 1 by scope, 7 by input.'); + expect(markdown).toContain('2 unsupported (1 declared, 1 quarantined)'); + expect(markdown).toContain('declared unsupported: Not executed.'); + }); + + test('the bounded summary carries the lane and exclusions', () => { + const summary = projectDesktopRunSummary(result(), path.join(os.tmpdir(), 'desktop-artifacts', 'run-id')); + + expect(summary.lane).toEqual(accessible); + expect(summary.excluded).toEqual(selection.excluded); + expect(summary.skippedReasons).toContain('declared unsupported: Not executed.'); + expect(Buffer.byteLength(JSON.stringify(summary))).toBeLessThanOrEqual(4 * 1024); + }); +}); describe('bounded desktop run summary', () => { test('keeps large suites and failure messages within the document limit', () => { diff --git a/packages/agentic/desktop-test/src/reporting/results.ts b/packages/agentic/desktop-test/src/reporting/results.ts index 2470c157bc..02fa673fe8 100644 --- a/packages/agentic/desktop-test/src/reporting/results.ts +++ b/packages/agentic/desktop-test/src/reporting/results.ts @@ -1,15 +1,28 @@ import path from 'node:path'; -import type { EventContext, ReportFailure, TestStatus } from './protocol.js'; +import type { EventContext, ReportFailure, SelectionExclusions, TestGap, TestStatus } from './protocol.js'; export type DesktopTestAttempt = { attempt: number; status: TestStatus; durationMs: number; reason?: string; + /** Present only on an `unsupported` attempt the producer declared in advance, never on a missing requirement. */ + gap?: TestGap; failure?: ReportFailure; context?: EventContext; }; +/** + * The input lane a run was asked to execute. It records the request and how the host was masked; it never claims what + * input a session delivered or enforced, hence the literal `policyEnforced: false`. + */ +export const desktopInputLaneSources = ['option', 'environment', 'driver'] as const; +/** `driver` means the lane was read from the attached driver instance's manifest instead of being requested here. */ +export type DesktopInputLaneSource = (typeof desktopInputLaneSources)[number]; +export type DesktopInputLaneReport = + | { input: 'accessible'; source: DesktopInputLaneSource; masking: 'host-masked' | 'verified-masked'; policyEnforced: false } + | { input: 'physical'; source: DesktopInputLaneSource; policyEnforced: false }; +export type DesktopSelectionReport = { excluded: SelectionExclusions }; export type DesktopTestResult = { testId: string; title: string; @@ -34,13 +47,22 @@ export type DesktopRunResult = { finishedAt: string; durationMs: number; context?: EventContext; + /** Set when the run executes one input lane. Lane runs treat undeclared skips and unsupported cases as non-success. */ + lane?: DesktopInputLaneReport; + /** What producers removed from their selection before execution. Excluded cases are not tests. */ + selection?: DesktopSelectionReport; counts: { selected: number; executed: number; passed: number; failed: number; skipped: number; + /** Every test whose final status is `unsupported`, declared or not. */ unsupported: number; + /** The subset of `unsupported` that carries a declared `unsupported` gap. */ + declaredUnsupported: number; + /** The subset of `unsupported` that carries a declared `quarantine` gap. */ + quarantined: number; incomplete: number; retries: number; flaky: number; @@ -59,6 +81,8 @@ export type DesktopRunSummary = { status: TestStatus; durationMs: number; context?: Pick; + lane?: DesktopInputLaneReport; + excluded?: SelectionExclusions; counts: DesktopRunResult['counts']; problems: { kind: 'test' | 'infrastructure' | 'unfinished'; @@ -122,7 +146,7 @@ export function projectDesktopRunSummary(result: DesktopRunResult, root: string) ...new Set( result.tests.flatMap((test) => test.attempts.flatMap((attempt) => - attempt.reason && ['skipped', 'unsupported'].includes(attempt.status) ? [clipText(attempt.reason, 256)] : [], + attempt.reason && ['skipped', 'unsupported'].includes(attempt.status) ? [clipText(describeReason(attempt), 256)] : [], ), ), ), @@ -152,6 +176,8 @@ export function projectDesktopRunSummary(result: DesktopRunResult, root: string) } : {}), counts: result.counts, + ...(result.lane ? { lane: result.lane } : {}), + ...(result.selection ? { excluded: result.selection.excluded } : {}), problems: problems.slice(0, SUMMARY_MAX_PROBLEMS), skippedReasons: skippedReasons.slice(0, SUMMARY_MAX_PROBLEMS), artifacts, @@ -205,6 +231,8 @@ export function summarizeDesktopRun( failed: 0, skipped: 0, unsupported: 0, + declaredUnsupported: 0, + quarantined: 0, incomplete: 0, retries: 0, flaky: 0, @@ -217,18 +245,26 @@ export function summarizeDesktopRun( else if (test.status === 'passed') counts.passed++; else if (test.status === 'failed') counts.failed++; else if (test.status === 'skipped') counts.skipped++; - else if (test.status === 'unsupported') counts.unsupported++; + else if (test.status === 'unsupported') { + counts.unsupported++; + const gap = test.attempts.at(-1)?.gap; + if (gap === 'unsupported') counts.declaredUnsupported++; + else if (gap === 'quarantine') counts.quarantined++; + } } const infrastructureFailed = input.failures.some(({ kind }) => kind !== 'test') || input.work.some(({ status }) => status === 'failed'); const unfinishedWork = input.work.some(({ status }) => ['not-run', 'incomplete', 'cancelled'].includes(status)); const noExecution = input.kind === 'tests' && counts.executed === 0; + // A lane run promised to execute its whole selection. Only a declared gap may leave a selected case unexecuted, and a + // run that executed nothing is never optional. + const laneUnmet = input.kind === 'tests' && input.lane !== undefined && counts.skipped + undeclaredUnsupported(counts) > 0; const status = infrastructureFailed || counts.failed > 0 ? 'failed' - : input.incomplete || unfinishedWork || counts.incomplete > 0 || (input.kind === 'tests' && counts.selected === 0) + : input.incomplete || unfinishedWork || counts.incomplete > 0 || laneUnmet || (input.kind === 'tests' && counts.selected === 0) ? 'incomplete' : noExecution - ? input.optional + ? input.optional && input.lane === undefined ? 'skipped' : 'incomplete' : 'passed'; @@ -236,6 +272,46 @@ export function summarizeDesktopRun( return { ...result, status, counts }; } +/** Unsupported cases nothing declared in advance: a requirement the target did not meet. */ +function undeclaredUnsupported(counts: Pick): number { + return counts.unsupported - counts.declaredUnsupported - counts.quarantined; +} + +const gapLabels = { unsupported: 'declared unsupported', quarantine: 'quarantined' } as const; + +/** A reason with its declared-gap label, so a declared gap never reads like a missing requirement. */ +function describeReason(attempt: Pick): string { + const reason = attempt.reason ?? ''; + return attempt.gap ? `${gapLabels[attempt.gap]}: ${reason}` : reason; +} + +/** `accessible (host-masked)`, `accessible (verified-masked)` or `physical pilot`; never a claim of enforced input. */ +export function describeDesktopInputLane(lane: DesktopInputLaneReport): string { + return lane.input === 'physical' ? 'physical pilot' : `accessible (${lane.masking})`; +} + +function describeLaneSource(lane: DesktopInputLaneReport): string { + return lane.source === 'option' ? '--input' : lane.source === 'environment' ? 'DESKTOP_INPUT' : 'the driver instance'; +} + +/** One line naming the lane and what each producer excluded before execution, or `undefined` when neither applies. */ +export function describeDesktopSelection(result: Pick): string | undefined { + const parts: string[] = []; + if (result.lane) parts.push(`input ${describeDesktopInputLane(result.lane)} from ${describeLaneSource(result.lane)}`); + if (result.selection) { + const { platform, scope, input } = result.selection.excluded; + parts.push(`excluded ${platform} by platform, ${scope} by scope, ${input} by input`); + } + return parts.length ? parts.join('; ') : undefined; +} + +/** The unsupported count with its declared and quarantined subsets, so undeclared gaps stand out. */ +export function describeDesktopUnsupported( + counts: Pick, +): string { + return `${counts.unsupported} unsupported (${counts.declaredUnsupported} declared, ${counts.quarantined} quarantined)`; +} + export function desktopRunExitCode(result: DesktopRunResult): number { return result.status === 'passed' || result.status === 'skipped' ? 0 : 1; } @@ -253,7 +329,7 @@ export function renderJUnit(result: DesktopRunResult): string { : test.status === 'passed' ? '' : ['skipped', 'unsupported'].includes(test.status) - ? `` + ? `` : ``; return `${body}${xml(history)}`; }); @@ -270,18 +346,34 @@ export function renderJUnit(result: DesktopRunResult): string { cases.push( '', ); - return `\n${cases.join('')}\n`; + const properties: [string, string][] = []; + if (result.lane) + properties.push( + ['desktop.input', result.lane.input], + ['desktop.input.label', describeDesktopInputLane(result.lane)], + ['desktop.input.source', result.lane.source], + ); + if (result.selection) + for (const [reason, count] of Object.entries(result.selection.excluded)) properties.push([`desktop.excluded.${reason}`, String(count)]); + if (result.counts.declaredUnsupported) properties.push(['desktop.declaredUnsupported', String(result.counts.declaredUnsupported)]); + if (result.counts.quarantined) properties.push(['desktop.quarantined', String(result.counts.quarantined)]); + const propertyXml = properties.length + ? `${properties.map(([name, value]) => ``).join('')}` + : ''; + return `\n${propertyXml}${cases.join('')}\n`; } export function renderRunSummary(result: DesktopRunResult): string { const c = result.counts; const reasons = [ - ...new Set(result.tests.flatMap((test) => test.attempts.flatMap((attempt) => (attempt.reason ? [attempt.reason] : [])))), + ...new Set(result.tests.flatMap((test) => test.attempts.flatMap((attempt) => (attempt.reason ? [describeReason(attempt)] : [])))), ]; + const selection = describeDesktopSelection(result); return ( `## Desktop tests: ${result.status}\n\n` + `Run: \`${result.runId}\`; endpoint: ${result.context?.endpoint ?? 'native'}; renderer: ${result.context?.renderer ?? 'not applicable'}.\n\n` + - `Selected ${c.selected}; executed ${c.executed}; passed ${c.passed}; failed ${c.failed}; skipped ${c.skipped}; unsupported ${c.unsupported}; incomplete ${c.incomplete}; retries ${c.retries}; flaky ${c.flaky}.\n\n` + + (selection ? `Selection: ${selection}.\n\n` : '') + + `Selected ${c.selected}; executed ${c.executed}; passed ${c.passed}; failed ${c.failed}; skipped ${c.skipped}; ${describeDesktopUnsupported(c)}; incomplete ${c.incomplete}; retries ${c.retries}; flaky ${c.flaky}.\n\n` + `Selected groups ${result.work.length}; not run ${result.work.filter(({ status }) => status === 'not-run').length}; incomplete ${result.work.filter(({ status }) => status === 'incomplete').length}.\n\n` + `Infrastructure/evidence/cleanup failures: ${result.failures.filter(({ kind }) => kind !== 'test').length}.\n\n` + (reasons.length diff --git a/packages/agentic/desktop-test/src/reporting/runQueries.test.ts b/packages/agentic/desktop-test/src/reporting/runQueries.test.ts index 75a352a566..b04dad2753 100644 --- a/packages/agentic/desktop-test/src/reporting/runQueries.test.ts +++ b/packages/agentic/desktop-test/src/reporting/runQueries.test.ts @@ -332,6 +332,95 @@ describe('persisted desktop run queries', () => { await expect(client.query({ operation: 'status', runId: reporter.runId })).rejects.toThrow(); }); + describe('input lane runs', () => { + const lane = { input: 'accessible', source: 'environment', masking: 'verified-masked', policyEnforced: false } as const; + let laneReporter: DesktopRunReporter; + let laneClient: DesktopRunQueryClient; + + beforeEach(async () => { + laneReporter = new DesktopRunReporter({ + artifactsRoot: root, + output: { write: () => true }, + reporterMode: 'json', + summary: false, + lane, + }); + service.registerRun({ runId: laneReporter.runId, root: laneReporter.root, expiresAt: Date.now() + 60_000 }); + laneClient = service.connect(service.issueGrant({ runIds: [laneReporter.runId], expiresAt: Date.now() + 60_000 })); + laneReporter.emit({ kind: 'selection', payload: { excluded: { platform: 1, scope: 2, input: 3 } } }); + laneReporter.emit({ kind: 'test-result', payload: { testId: 'ok', title: 'ok', attempt: 0, status: 'passed', durationMs: 1 } }); + laneReporter.emit({ + kind: 'test-result', + payload: { testId: 'gap', title: 'gap', attempt: 0, status: 'unsupported', durationMs: 0, reason: 'Declared.', gap: 'quarantine' }, + }); + await laneReporter.finish(); + }); + + function rewriteLane(change: (result: DesktopRunResult) => void): void { + const file = path.join(laneReporter.root, 'run.json'); + const result = JSON.parse(fs.readFileSync(file, 'utf8')) as DesktopRunResult; + change(result); + fs.writeFileSync(file, JSON.stringify(result)); + } + + test('reads back the lane, the exclusions and each declared gap', async () => { + expect(await laneClient.query({ operation: 'status', runId: laneReporter.runId })).toMatchObject({ + terminal: true, + status: 'passed', + lane, + excluded: { platform: 1, scope: 2, input: 3 }, + counts: { unsupported: 1, quarantined: 1, declaredUnsupported: 0 }, + }); + expect(await laneClient.query({ operation: 'tests', runId: laneReporter.runId })).toMatchObject({ + items: [ + { title: 'ok', status: 'passed' }, + { title: 'gap', status: 'unsupported', gap: 'quarantine', message: 'Declared.' }, + ], + }); + }); + + test('a declared gap edited out of the retained report no longer reads as a passing lane', async () => { + rewriteLane((result) => { + delete result.tests[1].attempts[0].gap; + }); + + expect(await laneClient.query({ operation: 'status', runId: laneReporter.runId })).toMatchObject({ + status: 'incomplete', + counts: { unsupported: 1, quarantined: 0, declaredUnsupported: 0 }, + }); + }); + + test('reads a lane taken from the driver instance and rejects an unknown source', async () => { + rewriteLane((result) => { + Object.assign(result, { lane: { ...lane, source: 'driver' } }); + }); + expect(await laneClient.query({ operation: 'status', runId: laneReporter.runId })).toMatchObject({ + lane: { ...lane, source: 'driver' }, + }); + + rewriteLane((result) => { + Object.assign(result, { lane: { ...lane, source: 'cli' } }); + }); + await expect(laneClient.query({ operation: 'status', runId: laneReporter.runId })).rejects.toThrow(); + }); + + test.each([ + ['a lane that claims enforcement', (result: DesktopRunResult) => Object.assign(result, { lane: { ...lane, policyEnforced: true } })], + ['a physical lane with masking', (result: DesktopRunResult) => Object.assign(result, { lane: { ...lane, input: 'physical' } })], + [ + 'a negative exclusion count', + (result: DesktopRunResult) => Object.assign(result, { selection: { excluded: { platform: -1, scope: 0, input: 0 } } }), + ], + ['a gap on a passed attempt', (result: DesktopRunResult) => Object.assign(result.tests[0].attempts[0], { gap: 'unsupported' })], + ])('rejects %s in a retained report', async (_name, change) => { + rewriteLane((result) => { + change(result); + }); + + await expect(laneClient.query({ operation: 'status', runId: laneReporter.runId })).rejects.toThrow(); + }); + }); + test('snapshot persistence failures remain reporting failures after the writer recovers', async () => { const write = fs.writeFileSync; const injected = jest.spyOn(fs, 'writeFileSync').mockImplementation((file, ...args) => { diff --git a/packages/agentic/desktop-test/src/reporting/runQueries.ts b/packages/agentic/desktop-test/src/reporting/runQueries.ts index e75938f236..1f6843a2be 100644 --- a/packages/agentic/desktop-test/src/reporting/runQueries.ts +++ b/packages/agentic/desktop-test/src/reporting/runQueries.ts @@ -4,10 +4,10 @@ import { createHash, randomUUID } from 'node:crypto'; import { isNativeError } from 'node:util/types'; import { desktopArtifactPath } from './collector.js'; -import { DesktopReportedError, reportStatuses, redactDesktopSecrets } from './protocol.js'; -import type { TestStatus } from './protocol.js'; -import { summarizeDesktopRun } from './results.js'; -import type { DesktopRunResult } from './results.js'; +import { DesktopReportedError, reportStatuses, redactDesktopSecrets, selectionExclusionReasons, testGaps } from './protocol.js'; +import type { SelectionExclusions, TestGap, TestStatus } from './protocol.js'; +import { desktopInputLaneSources, summarizeDesktopRun } from './results.js'; +import type { DesktopInputLaneReport, DesktopRunResult, DesktopSelectionReport } from './results.js'; export type DesktopRunQuery = | { operation: 'status'; runId: string } @@ -20,11 +20,15 @@ export type DesktopRunStatus = { terminal: boolean; status: TestStatus; counts: DesktopRunResult['counts']; + lane?: DesktopInputLaneReport; + excluded?: SelectionExclusions; }; export type DesktopRunQueryItem = { id: string; title: string; status?: TestStatus; + /** Set on an `unsupported` test whose gap was declared in advance. */ + gap?: TestGap; message?: string; }; export type DesktopRunPage = { @@ -176,7 +180,15 @@ export class DesktopRunQueryService { }); if (result.status === 'failed') canonical.status = 'failed'; if (input.operation === 'status') - return { schemaVersion: 1, runId: run.runId, terminal, status: canonical.status, counts: canonical.counts }; + return { + schemaVersion: 1, + runId: run.runId, + terminal, + status: canonical.status, + counts: canonical.counts, + ...(canonical.lane ? { lane: canonical.lane } : {}), + ...(canonical.selection ? { excluded: canonical.selection.excluded } : {}), + }; let offset = 0; if (input.cursor) { const cursor = grant.cursors.get(input.cursor); @@ -225,6 +237,7 @@ export class DesktopRunQueryService { id: item.id, title: title.text, ...(item.status ? { status: item.status } : {}), + ...(item.gap ? { gap: item.gap } : {}), ...(message ? { message: message.text } : {}), }, omitted: title.omitted + (message?.omitted ?? 0), @@ -239,6 +252,7 @@ export class DesktopRunQueryService { id: id('test', test.testId), title: test.title, status: test.status, + ...(test.attempts.at(-1)?.gap ? { gap: test.attempts.at(-1)!.gap } : {}), message: test.attempts.at(-1)?.failure?.error.message ?? test.attempts.at(-1)?.reason, })); case 'work': @@ -346,11 +360,15 @@ function parseResult(text: string, runId: string): DesktopRunResult & { terminal throw new TypeError('Invalid persisted desktop attempt duration.'); const reason = attempt.reason; if (reason !== undefined) textField(reason); + const gap = attempt.gap; + if (gap !== undefined && (!testGaps.includes(gap as TestGap) || attempt.status !== 'unsupported')) + throw new TypeError('Invalid persisted desktop test gap.'); return { attempt: attempt.attempt, durationMs: attempt.durationMs, status: attempt.status, ...(typeof reason === 'string' ? { reason } : {}), + ...(gap !== undefined ? { gap: gap as TestGap } : {}), ...(attempt.failure !== undefined ? { failure: new DesktopReportedError(attempt.failure).failure } : {}), }; }); @@ -375,6 +393,8 @@ function parseResult(text: string, runId: string): DesktopRunResult & { terminal textField(value.finishedAt); if (typeof value.durationMs !== 'number' || !Number.isFinite(value.durationMs) || value.durationMs < 0) throw new TypeError('Invalid persisted desktop run duration.'); + const lane = laneField(value.lane); + const selection = selectionField(value.selection); const result = summarizeDesktopRun({ schemaVersion: 1, runId, @@ -382,6 +402,8 @@ function parseResult(text: string, runId: string): DesktopRunResult & { terminal startedAt: value.startedAt, finishedAt: value.finishedAt, durationMs: value.durationMs, + ...(lane ? { lane } : {}), + ...(selection ? { selection } : {}), tests, work, failures, @@ -390,6 +412,30 @@ function parseResult(text: string, runId: string): DesktopRunResult & { terminal return { ...result, status: value.status, ...(typeof value.terminal === 'boolean' ? { terminal: value.terminal } : {}) }; } +function laneField(value: unknown): DesktopInputLaneReport | undefined { + if (value === undefined) return undefined; + const lane = record(value); + const source = desktopInputLaneSources.find((candidate) => candidate === lane.source); + if (lane.policyEnforced !== false || source === undefined) throw new TypeError('Invalid persisted desktop input lane.'); + if (lane.input === 'physical' && lane.masking === undefined) return { input: 'physical', source, policyEnforced: false }; + if (lane.input === 'accessible' && (lane.masking === 'host-masked' || lane.masking === 'verified-masked')) + return { input: 'accessible', source, masking: lane.masking, policyEnforced: false }; + throw new TypeError('Invalid persisted desktop input lane.'); +} + +function selectionField(value: unknown): DesktopSelectionReport | undefined { + if (value === undefined) return undefined; + const excluded = record(record(value).excluded); + const counts = {} as SelectionExclusions; + for (const reason of selectionExclusionReasons) { + const count = excluded[reason]; + if (typeof count !== 'number' || !Number.isSafeInteger(count) || count < 0) + throw new TypeError('Invalid persisted desktop exclusion count.'); + counts[reason] = count; + } + return { excluded: counts }; +} + function record(value: unknown): Record { if (!value || typeof value !== 'object' || Array.isArray(value)) throw new TypeError('Expected a desktop run query object.'); return value as Record; diff --git a/packages/agentic/desktop-test/src/reporting/structured.test.ts b/packages/agentic/desktop-test/src/reporting/structured.test.ts index 1a7f5b1760..74a2c7f3d8 100644 --- a/packages/agentic/desktop-test/src/reporting/structured.test.ts +++ b/packages/agentic/desktop-test/src/reporting/structured.test.ts @@ -20,6 +20,24 @@ describe('structured desktop events', () => { expect(includeDesktopEvent(event, 'milestones', 'service')).toBe(true); }); + test('exclusion counts are progress detail and project as plain numbers', () => { + const event = makeEvent({ kind: 'selection', payload: { excluded: { platform: 1, scope: 0, input: 2 } } }); + + expect(includeDesktopEvent(event, 'milestones')).toBe(false); + expect(includeDesktopEvent(event, 'progress')).toBe(true); + expect(projectDesktopEvent(event).payload).toEqual({ excluded: { platform: 1, scope: 0, input: 2 } }); + }); + + test('a declared gap stays visible at every level and projects with its reason', () => { + const event = makeEvent({ + kind: 'test-result', + payload: { testId: 'test', title: 'gap', attempt: 0, status: 'unsupported', durationMs: 0, reason: 'Declared.', gap: 'quarantine' }, + }); + + expect(includeDesktopEvent(event, 'milestones')).toBe(true); + expect(projectDesktopEvent(event).payload).toMatchObject({ status: 'unsupported', gap: 'quarantine', reason: 'Declared.' }); + }); + test('projects and bounds large event text while retaining canonical IDs', () => { const event = makeEvent({ kind: 'output', diff --git a/packages/agentic/desktop-test/src/reporting/structured.ts b/packages/agentic/desktop-test/src/reporting/structured.ts index 8e89133ea6..544be9ebc9 100644 --- a/packages/agentic/desktop-test/src/reporting/structured.ts +++ b/packages/agentic/desktop-test/src/reporting/structured.ts @@ -41,6 +41,7 @@ export function includeDesktopEvent(event: DesktopEvent, level: DesktopEventLeve return event.payload.status !== 'passed'; case 'work-selected': case 'test-start': + case 'selection': case 'output': return false; } @@ -88,8 +89,12 @@ export function projectDesktopEvent(event: CollectedDesktopEvent): DesktopEventR payload.status = event.payload.status; payload.durationMs = event.payload.durationMs; if (event.payload.reason) payload.reason = clip(event.payload.reason); + if (event.payload.gap) payload.gap = event.payload.gap; if (event.payload.failure) payload.failure = projectFailure(event.payload.failure); break; + case 'selection': + payload.excluded = { ...event.payload.excluded }; + break; case 'artifact': payload.path = clip(event.payload.path); payload.description = clip(event.payload.description); diff --git a/packages/agentic/desktop-test/src/supervisor.test.ts b/packages/agentic/desktop-test/src/supervisor.test.ts new file mode 100644 index 0000000000..35ced5e4cf --- /dev/null +++ b/packages/agentic/desktop-test/src/supervisor.test.ts @@ -0,0 +1,71 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { accessibleLaneMaskedFeatures } from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; +import type { ResolvedDesktopTarget } from './config.js'; +import { startNativeDesktopService } from './driverService.js'; +import { DesktopRunReporter } from './reporting/collector.js'; +import { DesktopSupervisor } from './supervisor.js'; + +jest.mock('./driverService', () => ({ startDesktopService: jest.fn(), startNativeDesktopService: jest.fn() })); +jest.mock('./adapters', () => ({ resolveDesktopApplication: jest.fn(async () => ({ executablePath: 'Fixture.app' })) })); + +describe('desktop supervisor input lane', () => { + const platform = Object.getOwnPropertyDescriptor(process, 'platform')!; + const startNative = startNativeDesktopService as jest.MockedFunction; + let artifactsRoot: string; + + beforeEach(() => { + artifactsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-desktop-supervisor-')); + // The helper is mocked, so any host can stand in for the macOS machine the endpoint requires. + Object.defineProperty(process, 'platform', { ...platform, value: 'darwin' }); + startNative.mockReset(); + // eslint-disable-next-line @microsoft/sdl/no-insecure-url -- the mocked service stands in for a loopback Desktop Driver + startNative.mockResolvedValue({ url: 'http://127.0.0.1:1', close: async () => undefined } as Awaited< + ReturnType + >); + }); + + afterEach(() => { + Object.defineProperty(process, 'platform', platform); + fs.rmSync(artifactsRoot, { force: true, recursive: true }); + }); + + async function nativeServiceOptions(input?: DesktopInputLane) { + const output = { write: () => true }; + const reporter = new DesktopRunReporter({ artifactsRoot, errorOutput: output, output, reporterMode: 'plain', summary: false }); + const target: ResolvedDesktopTarget = { + endpoint: 'macos', + id: 'fixture', + instanceId: 'fixture-1', + projectRoot: artifactsRoot, + configPath: path.join(artifactsRoot, 'desktop.config.mts'), + host: { kind: 'installed' }, + }; + const supervisor = new DesktopSupervisor({ + target, + reporter, + reportSession: { reporter, delegated: true, environment: {}, close: async () => 0 }, + ...(input ? { input } : {}), + }); + try { + await supervisor.start(); + } finally { + await supervisor.close(); + } + expect(startNative).toHaveBeenCalledTimes(1); + return startNative.mock.calls[0][0]; + } + + test('starts the native host with physical input disabled for an accessible lane', async () => { + const options = await nativeServiceOptions('accessible'); + expect(options.disabledInputFeatures).toEqual([...accessibleLaneMaskedFeatures]); + }); + + test.each([['physical' as const], [undefined]])('leaves the native host unmasked for %s', async (input) => { + const options = await nativeServiceOptions(input); + expect(options).not.toHaveProperty('disabledInputFeatures'); + }); +}); diff --git a/packages/agentic/desktop-test/src/supervisor.ts b/packages/agentic/desktop-test/src/supervisor.ts index 4249c82ea6..eeb3766491 100644 --- a/packages/agentic/desktop-test/src/supervisor.ts +++ b/packages/agentic/desktop-test/src/supervisor.ts @@ -4,6 +4,8 @@ import path from 'node:path'; import { createHash } from 'node:crypto'; import type { DesktopDriverServer, DesktopHost } from '@fluentui-react-native/desktop-driver'; +import { accessibleLaneMaskedFeatures } from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; import type { DesktopAgentConnectionOptions } from './agentConnection.js'; import { startDesktopService, startNativeDesktopService } from './driverService.js'; import type { DesktopService, DesktopServiceOptions } from './driverService.js'; @@ -32,6 +34,11 @@ export type DesktopSupervisorOptions = { */ agent?: DesktopAgentConnectionOptions; recovery?: DesktopServiceOptions['recovery']; + /** + * The input lane of a `test` run. An accessible lane starts the native host with keyboard, physical click and wheel + * disabled. A host supplied through `hostFactory` is not masked here, so the suite verifies it before running anything. + */ + input?: DesktopInputLane; }; export class DesktopSupervisor { @@ -147,6 +154,7 @@ export class DesktopSupervisor { if (application.leaseNonce) this.reporter.addSecret(application.leaseNonce); return startNativeDesktopService({ ...options, + ...(this.options.input === 'accessible' ? { disabledInputFeatures: accessibleLaneMaskedFeatures } : {}), application, nativeDriver: this.target.nativeDriver, endpoint: this.target.endpoint, diff --git a/packages/agentic/desktop-test/src/wdio/configuration.ts b/packages/agentic/desktop-test/src/wdio/configuration.ts index 506294ad4f..f16768c542 100644 --- a/packages/agentic/desktop-test/src/wdio/configuration.ts +++ b/packages/agentic/desktop-test/src/wdio/configuration.ts @@ -2,7 +2,11 @@ import { pathToFileURL } from 'node:url'; import { randomUUID } from 'node:crypto'; import { installDesktopWebdriverCommands } from '@fluentui-react-native/desktop-driver/wdio'; import type { DesktopEndpoint, DesktopRenderer } from '@fluentui-react-native/desktop-driver'; +import { resolveDesktopClickMode } from '@fluentui-react-native/desktop-driver/authoring'; import type { DesktopReadiness } from '../config.js'; +import { desktopJasmineRegistry } from '../jasmine/registry.js'; +import { desktopTestSelectionFromEnvironment } from '../jasmine/selection.js'; +import { currentDesktopJasmineEnvironment, selectDesktopSpecs } from '../jasmine/specSelection.js'; import { connectReportProducer } from '../reporting/transport.js'; import { DesktopReportedError, serializeReportError } from '../reporting/protocol.js'; @@ -56,6 +60,8 @@ export async function createDesktopWdioConfiguration(settings: DesktopWdioSettin } const address = new URL(settings.url); const readiness = settings.readiness; + // The launcher read the selection too, so a malformed one fails the run before any worker starts. + const selection = desktopTestSelectionFromEnvironment(); const configuration = { ...hooks, runner: 'local', @@ -88,7 +94,7 @@ export async function createDesktopWdioConfiguration(settings: DesktopWdioSettin platformName: settings.endpoint === 'macos' ? 'macos' : 'windows', 'furn:target': settings.targetId, 'furn:launchMode': settings.ownership === 'external' ? 'attach' : 'launch', - 'furn:clickMode': 'auto', + 'furn:clickMode': resolveDesktopClickMode(undefined, 'auto', selection?.lane), }, ], beforeSession: async (...args: unknown[]) => { @@ -97,6 +103,10 @@ export async function createDesktopWdioConfiguration(settings: DesktopWdioSettin }, before: async (capabilities: unknown, specs: unknown, browser: WebdriverIO.Browser) => { installDesktopWebdriverCommands(browser, settings.url); + // Before the application is touched: a lane that cannot vouch for its tests must stop here. + if (settings.framework === 'jasmine') { + selectDesktopSpecs(desktopJasmineRegistry(), currentDesktopJasmineEnvironment(), browser.capabilities['furn:features']); + } if ((await browser.getWindowHandles()).length === 0) throw new Error('Native application did not expose a window.'); if (readiness?.testId) { await browser.$(`~${readiness.testId}`).waitForDisplayed({ diff --git a/packages/agentic/desktop-test/src/wdio/reporter.ts b/packages/agentic/desktop-test/src/wdio/reporter.ts index ab5b3df4e4..7379061b54 100644 --- a/packages/agentic/desktop-test/src/wdio/reporter.ts +++ b/packages/agentic/desktop-test/src/wdio/reporter.ts @@ -1,6 +1,8 @@ import { createHash, randomUUID } from 'node:crypto'; import WDIOReporter from '@wdio/reporter'; import type { HookStats, RunnerStats, TestStats } from '@wdio/reporter'; +import { desktopJasmineRegistry, excludedSpecCounts } from '../jasmine/registry.js'; +import type { DesktopSpecEntry } from '../jasmine/registry.js'; import { connectReportProducer } from '../reporting/transport.js'; import type { ReportProducer } from '../reporting/transport.js'; import { serializeReportError } from '../reporting/protocol.js'; @@ -71,6 +73,7 @@ export default class DesktopWdioReporter extends WDIOReporter { } override onTestStart(test: TestStats): void { + if (this.hidden(test)) return; this.report( { kind: 'test-start', payload: { testId: this.testId(test), title: test.fullTitle, attempt: this.attempt(test) } }, { ...this.context, test: test.uid }, @@ -90,13 +93,29 @@ export default class DesktopWdioReporter extends WDIOReporter { this.testResult(test); } + /** The spec the Jasmine adapter registered for a test, when `desktopIt` declared it. */ + private entry(test: TestStats): DesktopSpecEntry | undefined { + return desktopJasmineRegistry().entries.get(test.uid); + } + + /** An excluded case is not a test, and a blocked run reports its fault rather than every spec it disabled. */ + private hidden(test: TestStats): boolean { + return desktopJasmineRegistry().blocked || this.entry(test)?.kind === 'excluded'; + } + private testResult(test: TestStats): void { + if (this.hidden(test)) return; const testId = this.testId(test); const attempt = this.attempt(test); const key = `${testId}/${attempt}`; if (this.endedTests.has(key)) return; this.endedTests.add(key); - const status = test.state === 'pending' ? 'skipped' : test.state; + const entry = this.entry(test); + // A spec the adapter replaced or filtered out finishes as skipped or pending and never ran, whichever the framework calls it. + const notExecuted = test.state === 'pending' || test.state === 'skipped'; + const gap = notExecuted && entry?.kind === 'gap' ? entry.gap : undefined; + const missing = notExecuted && entry?.kind === 'run' ? entry.missing : undefined; + const status = gap || missing?.length ? 'unsupported' : test.state === 'pending' ? 'skipped' : test.state; this.report( { kind: 'test-result', @@ -106,6 +125,8 @@ export default class DesktopWdioReporter extends WDIOReporter { attempt, status, durationMs: test.duration, + ...(gap ? { gap: gap.kind, reason: gap.reason } : {}), + ...(missing?.length ? { reason: `Missing desktop requirements: ${missing.join(', ')}.` } : {}), ...(status === 'skipped' ? { reason: test.pendingReason || 'Skipped by the test framework.' } : {}), ...(status === 'failed' ? { failure: this.failure(`${key}/failure`, test.error ?? new AggregateError(test.errors ?? [], 'WDIO test failed.'), 'test') } @@ -143,6 +164,11 @@ export default class DesktopWdioReporter extends WDIOReporter { failure: this.failure(`${this.scopeId}/runner`, new Error(runner.error), 'infrastructure'), }, }); + const excluded = excludedSpecCounts(desktopJasmineRegistry()); + // Keyed by the spec group, so a retried attempt that announces the same exclusions again is counted once. + if (excluded.platform + excluded.scope + excluded.input > 0) { + this.report({ kind: 'selection', payload: { excluded, key: this.groupId } }); + } this.report({ kind: 'scope-end', payload: { status: runner.failures || runner.error ? 'failed' : 'passed' } }); this.schedule((producer) => producer.finish(runner.failures || runner.error ? 'failed' : 'passed')); } diff --git a/packages/agentic/desktop-test/src/wdio/run.ts b/packages/agentic/desktop-test/src/wdio/run.ts index c86eeb5efe..e9db1b7bbf 100644 --- a/packages/agentic/desktop-test/src/wdio/run.ts +++ b/packages/agentic/desktop-test/src/wdio/run.ts @@ -2,15 +2,32 @@ import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { resolve } from 'import-meta-resolve'; +import type { DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; import type { ResolvedDesktopConfig, DesktopWdioSuite } from '../config.js'; -import { resolveDesktopSpecs } from '../config.js'; +import { resolveDesktopSpecs, resolveDesktopTestScope } from '../config.js'; +import { desktopTestSelectionEnvironment, serializeDesktopTestSelection } from '../jasmine/selection.js'; import type { DesktopSupervisor } from '../supervisor.js'; import type { DesktopWdioSettings } from './configuration.js'; import { wdioGroupId } from './identity.js'; -export async function runDesktopWdioSuite(supervisor: DesktopSupervisor, config: ResolvedDesktopConfig, suiteName: string): Promise { +/** A lane selects tests through the Jasmine adapter, so a suite on another framework cannot honor one. */ +export function assertDesktopSuiteSupportsInput(suiteName: string, suite: DesktopWdioSuite, input: DesktopInputLane | undefined): void { + if (input && suite.framework !== 'jasmine') { + throw Object.assign(new TypeError(`--input ${input} requires a jasmine suite; "${suiteName}" uses ${suite.framework}.`), { + code: 'config', + }); + } +} + +export async function runDesktopWdioSuite( + supervisor: DesktopSupervisor, + config: ResolvedDesktopConfig, + suiteName: string, + input?: DesktopInputLane, +): Promise { const suite = config.config.suites?.[suiteName]; if (!suite || suite.kind !== 'wdio') throw new Error(`No standalone WDIO suite "${suiteName}" is configured.`); + assertDesktopSuiteSupportsInput(suiteName, suite, input); const specs = resolveDesktopSpecs(suite, supervisor.target.endpoint, config.projectRoot); const workIds: string[] = []; for (const selected of specs) { @@ -58,12 +75,19 @@ export async function runDesktopWdioSuite(supervisor: DesktopSupervisor, config: { mode: 0o600 }, ); const launcher = modulePath('@wdio/cli'); + // The selection travels as data in the environment: a worker parses it, and nothing is spliced into code that runs. + const selection = serializeDesktopTestSelection({ + endpoint: supervisor.target.endpoint, + ...(input ? { lane: input } : {}), + testScopes: config.config.testScopes, + values: resolveDesktopTestScope(config.testScopes, supervisor.target), + }); await supervisor.worker( { command: process.execPath, args: [fileURLToPath(new URL('../../config/launcher.cjs', import.meta.url)), pathToFileURL(launcher).href, configPath], cwd: config.projectRoot, - env: { E2ETEST_PLATFORM: supervisor.target.endpoint, NATIVE_TESTING: 'false' }, + env: { E2ETEST_PLATFORM: supervisor.target.endpoint, NATIVE_TESTING: 'false', [desktopTestSelectionEnvironment]: selection }, label: `WDIO ${suiteName} (${supervisor.target.endpoint})`, }, suiteDeadline(suite, specs.length), diff --git a/packages/agentic/native-core/references/contributing.md b/packages/agentic/native-core/references/contributing.md index abd2c0a237..f0e813a66a 100644 --- a/packages/agentic/native-core/references/contributing.md +++ b/packages/agentic/native-core/references/contributing.md @@ -55,6 +55,15 @@ full catalog before running the selected native tests. Add `--paper` to run the macOS Paper lane; a successful Paper build alone does not establish Paper runtime or input parity. +Executable cases are declared with `desktopIt` and its `requires` list (see +`@fluentui-react-native/storybook-desktop/references/story-tests.md`), not with run-time +`skip` guards. Cases that click or type declare `physical-click` and `keyboard`, which +derive the `physical` input lane; `focus` only asks the host to report focus support. +Only the placement cases declare no requirement and run in both lanes. The Win32 +Tab-exit behavior (#4377) is a declared `unsupported` gap, not a skip. Add +`--input accessible` or `--input physical` to select a lane: `accessible` selects only +the placement cases, and `physical` selects every case. + ## Apple boundary probe After building macOS Storybook, run the reproducible Apple boundary probe: diff --git a/packages/agentic/native-core/src/components/callout/Callout.stories.tsx b/packages/agentic/native-core/src/components/callout/Callout.stories.tsx index 2056a387f9..4010ef6020 100644 --- a/packages/agentic/native-core/src/components/callout/Callout.stories.tsx +++ b/packages/agentic/native-core/src/components/callout/Callout.stories.tsx @@ -4,6 +4,7 @@ import { Pressable, Text, View, StyleSheet } from 'react-native'; import type { Meta, StoryObj } from '@storybook/react-native'; import { useFocusablePressable } from '@fluentui-react-native/framework-base'; import type { PointerFocusMode } from '@fluentui-react-native/framework-base'; +import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; import type { WdioStory } from '@fluentui-react-native/storybook-desktop/testing'; import { Callout } from './Callout'; @@ -344,83 +345,78 @@ export default meta; type Story = WdioStory>; export const Default: Story = { tags: ['desktop-e2e'], - wdio: { - 'light-dismisses on an outside owner click and explicitly rearms': async ({ browser, expect, platform, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Light dismissal requires physical owner input.'); - return; - } - const status = await browser.$('~modern-callout-status'); - const dismissal = await browser.$('~modern-callout-dismissal'); - // The App SDK notification itself has no cause; its unclassified path can win the outside-pointer race. - const expected = platform === 'windows' ? /^Dismissed: (outside-pointer|native-light-dismiss)$/ : 'Dismissed: outside-pointer'; - for (let attempt = 0; attempt < 2; attempt++) { + wdio: [ + desktopIt( + 'light-dismisses on an outside owner click and explicitly rearms', + { requires: ['physical-click'] }, + async ({ browser, expect, platform }) => { + const status = await browser.$('~modern-callout-status'); + const dismissal = await browser.$('~modern-callout-dismissal'); + // The App SDK notification itself has no cause; its unclassified path can win the outside-pointer race. + const expected = platform === 'windows' ? /^Dismissed: (outside-pointer|native-light-dismiss)$/ : 'Dismissed: outside-pointer'; + for (let attempt = 0; attempt < 2; attempt++) { + await expect(status).toHaveText('Ready'); + await expect(dismissal).toHaveText('Open'); + await status.click(); + await expect(dismissal).toHaveText(expected); + // A latched presentation must not reopen merely because open remains true; readiness would reset the label. + await browser.pause(500); + await expect(dismissal).toHaveText(expected); + await (await browser.$('~modern-callout-reopen')).click(); + } await expect(status).toHaveText('Ready'); await expect(dismissal).toHaveText('Open'); - await status.click(); - await expect(dismissal).toHaveText(expected); - // A latched presentation must not reopen merely because open remains true; readiness would reset the label. + }, + ), + desktopIt( + 'keeps the presentation open for pointer input on its view anchor', + { requires: ['physical-click'] }, + async ({ browser, expect, platform }) => { + await expect(await browser.$('~modern-callout-status')).toHaveText('Ready'); + const dismissal = await browser.$('~modern-callout-dismissal'); + await expect(dismissal).toHaveText('Open'); + await (await browser.$('~modern-callout-anchor')).click(); await browser.pause(500); - await expect(dismissal).toHaveText(expected); - await (await browser.$('~modern-callout-reopen')).click(); - } - await expect(status).toHaveText('Ready'); - await expect(dismissal).toHaveText('Open'); - }, - 'keeps the presentation open for pointer input on its view anchor': async ({ browser, expect, platform, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Anchor input requires physical owner input.'); - return; - } - await expect(await browser.$('~modern-callout-status')).toHaveText('Ready'); - const dismissal = await browser.$('~modern-callout-dismissal'); - await expect(dismissal).toHaveText('Open'); - await (await browser.$('~modern-callout-anchor')).click(); - await browser.pause(500); - await expect(dismissal).toHaveText('Open'); - await (await browser.$('~modern-callout-status')).click(); - await expect(dismissal).toHaveText( - platform === 'windows' ? /^Dismissed: (outside-pointer|native-light-dismiss)$/ : 'Dismissed: outside-pointer', - ); - }, - 'places content below the view anchor with the requested gap': async (context) => { + await expect(dismissal).toHaveText('Open'); + await (await browser.$('~modern-callout-status')).click(); + await expect(dismissal).toHaveText( + platform === 'windows' ? /^Dismissed: (outside-pointer|native-light-dismiss)$/ : 'Dismissed: outside-pointer', + ); + }, + ), + desktopIt('places content below the view anchor with the requested gap', async (context) => { const { expectCalloutPlacement } = await import('./placement.wdio.ts'); await expectCalloutPlacement(context); - }, - }, + }), + ], }; export const NativeOperations: Story = { args: { exercise: 'commands' }, tags: ['desktop-e2e'], - wdio: { - 'readies, confirms descendant focus, closes, and explicitly rearms': async ({ browser, expect, platform, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Physical owner input unavailable.'); - return; - } - const status = await browser.$('~modern-callout-status'); - const operations = await browser.$('~modern-callout-operations'); - await expect(status).toHaveText('Ready'); - for (let attempt = 0; attempt < 2; attempt++) { - await (await browser.$('~modern-callout-reopen')).click(); - await expect(operations).toHaveText('confirmed/confirmed/confirmed'); - await expect(await browser.$('~modern-callout-dismissal')).toHaveText('Dismissed: programmatic'); - await expect(await browser.$('~modern-callout-focus-status')).toHaveText('Child focused'); - // Windows popups are owned tool windows that the driver does not enumerate as top-level windows. - if (platform === 'macos') await browser.waitUntil(async () => (await browser.getWindowHandles()).length === 1, { timeout: 5000 }); - } - }, - }, + wdio: [ + desktopIt( + 'readies, confirms descendant focus, closes, and explicitly rearms', + { requires: ['physical-click'] }, + async ({ browser, expect, platform }) => { + const status = await browser.$('~modern-callout-status'); + const operations = await browser.$('~modern-callout-operations'); + await expect(status).toHaveText('Ready'); + for (let attempt = 0; attempt < 2; attempt++) { + await (await browser.$('~modern-callout-reopen')).click(); + await expect(operations).toHaveText('confirmed/confirmed/confirmed'); + await expect(await browser.$('~modern-callout-dismissal')).toHaveText('Dismissed: programmatic'); + await expect(await browser.$('~modern-callout-focus-status')).toHaveText('Child focused'); + // Windows popups are owned tool windows that the driver does not enumerate as top-level windows. + if (platform === 'macos') await browser.waitUntil(async () => (await browser.getWindowHandles()).length === 1, { timeout: 5000 }); + } + }, + ), + ], }; export const KeyboardDismissal: Story = { args: { exercise: 'focus' }, - wdio: { - 'dismisses focused content on Escape': async ({ browser, expect, skip }) => { - const features = browser.capabilities['furn:features']; - if (!features?.physicalClick || !features.keyboard) { - skip('Escape dismissal requires physical owner and keyboard input.'); - return; - } + wdio: [ + desktopIt('dismisses focused content on Escape', { requires: ['physical-click', 'keyboard'] }, async ({ browser, expect }) => { const status = await browser.$('~modern-callout-status'); await expect(status).toHaveText('Ready'); await (await browser.$('~modern-callout-reopen')).click(); @@ -430,195 +426,186 @@ export const KeyboardDismissal: Story = { await expect(dismissal).toHaveText('Open'); await browser.keys('\uE00C'); await expect(dismissal).toHaveText('Dismissed: escape'); - }, - }, + }), + ], }; export const FirstFocusable: Story = { render: () => , tags: ['desktop-e2e'], - wdio: { - 'confirms the first eligible content without a FocusZone': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Entry focus requires physical owner activation.'); - return; - } + wdio: [ + desktopIt('confirms the first eligible content without a FocusZone', { requires: ['physical-click'] }, async ({ browser, expect }) => { await (await browser.$('~entry-callout-open')).click(); await expect(await browser.$('~entry-callout-status')).toHaveText('confirmed'); await expect(await browser.$('~entry-callout-focused')).toHaveText('First'); - }, - }, + }), + ], }; export const LastFocusable: Story = { render: () => , tags: ['desktop-e2e'], - wdio: { - 'confirms the last eligible nested content without a FocusZone': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Entry focus requires physical owner activation.'); - return; - } - await (await browser.$('~entry-callout-open')).click(); - await expect(await browser.$('~entry-callout-status')).toHaveText('confirmed'); - await expect(await browser.$('~entry-callout-focused')).toHaveText('Last'); - }, - }, + wdio: [ + desktopIt( + 'confirms the last eligible nested content without a FocusZone', + { requires: ['physical-click'] }, + async ({ browser, expect }) => { + await (await browser.$('~entry-callout-open')).click(); + await expect(await browser.$('~entry-callout-status')).toHaveText('confirmed'); + await expect(await browser.$('~entry-callout-focused')).toHaveText('Last'); + }, + ), + ], }; export const EmptyFocus: Story = { render: () => , tags: ['desktop-e2e'], - wdio: { - 'reports missing eligible content instead of manufacturing focus confirmation': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Entry focus requires physical owner activation.'); - return; - } - await (await browser.$('~entry-callout-open')).click(); - await expect(await browser.$('~entry-callout-status')).toHaveText('not-mounted'); - await expect(await browser.$('~entry-callout-focused')).toHaveText('None'); - }, - }, + wdio: [ + desktopIt( + 'reports missing eligible content instead of manufacturing focus confirmation', + { requires: ['physical-click'] }, + async ({ browser, expect }) => { + await (await browser.$('~entry-callout-open')).click(); + await expect(await browser.$('~entry-callout-status')).toHaveText('not-mounted'); + await expect(await browser.$('~entry-callout-focused')).toHaveText('None'); + }, + ), + ], }; export const ContainerFocus: Story = { render: () => , tags: ['desktop-e2e'], - wdio: { - 'confirms an explicitly focusable empty content container': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Container focus requires physical owner activation.'); - return; - } + wdio: [ + desktopIt('confirms an explicitly focusable empty content container', { requires: ['physical-click'] }, async ({ browser, expect }) => { await (await browser.$('~entry-callout-open')).click(); await expect(await browser.$('~entry-callout-status')).toHaveText('confirmed'); await expect(await browser.$('~entry-callout-focused')).toHaveText('Container'); - }, - }, + }), + ], }; export const DeferredContainerFocus: Story = { render: () => , tags: ['desktop-e2e'], - wdio: { - 'orders a focus request after the content update committed before it': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Container focus requires physical owner activation.'); - return; - } - await (await browser.$('~entry-callout-open')).click(); - await expect(await browser.$('~entry-callout-status')).toHaveText('confirmed'); - await expect(await browser.$('~entry-callout-focused')).toHaveText('Container'); - }, - }, + wdio: [ + desktopIt( + 'orders a focus request after the content update committed before it', + { requires: ['physical-click'] }, + async ({ browser, expect }) => { + await (await browser.$('~entry-callout-open')).click(); + await expect(await browser.$('~entry-callout-status')).toHaveText('confirmed'); + await expect(await browser.$('~entry-callout-focused')).toHaveText('Container'); + }, + ), + ], }; export const LocalRectangle: Story = { args: { anchorKind: 'rect' }, - wdio: { - 'places content below the local rectangle with the requested gap': async (context) => { + wdio: [ + desktopIt('places content below the local rectangle with the requested gap', async (context) => { const { expectCalloutPlacement } = await import('./placement.wdio.ts'); await expectCalloutPlacement(context, { x: 40, y: 30, height: 30 }); - }, - }, + }), + ], }; export const LocalPoint: Story = { args: { anchorKind: 'point' }, - wdio: { - 'places content below the local point with the requested gap': async (context) => { + wdio: [ + desktopIt('places content below the local point with the requested gap', async (context) => { const { expectCalloutPlacement } = await import('./placement.wdio.ts'); await expectCalloutPlacement(context, { x: 40, y: 60, height: 0 }); - }, - }, + }), + ], }; export const MenuComposition: Story = { render: ({ pointerFocus }) => , args: { pointerFocus: 'platform' }, argTypes: { pointerFocus: { control: 'select', options: ['platform', 'always', 'never'] } }, tags: ['desktop-e2e'], - wdio: { - 'enters a FocusZone, navigates by arrow, and follows the platform focus convention on Escape': async ({ - browser, - expect, - platform, - skip, - }) => { - const { exerciseMenuComposition } = await import('./menuComposition.wdio.ts'); - // Windows and Win32 focus a pressed trigger, so closing the menu restores it. AppKit leaves keyboard focus where - // it was on click, so macOS restores the owner window's previous first responder instead. - await exerciseMenuComposition({ browser, expect, skip }, { restoresTrigger: platform !== 'macos' }); - }, - }, + wdio: [ + desktopIt( + 'enters a FocusZone, navigates by arrow, and follows the platform focus convention on Escape', + { requires: ['physical-click', 'keyboard', 'focus'] }, + async ({ browser, expect, platform }) => { + const { exerciseMenuComposition } = await import('./menuComposition.wdio.ts'); + // Windows and Win32 focus a pressed trigger, so closing the menu restores it. AppKit leaves keyboard focus where + // it was on click, so macOS restores the owner window's previous first responder instead. + await exerciseMenuComposition({ browser, expect }, { restoresTrigger: platform !== 'macos' }); + }, + ), + ], }; export const TabContinuation: Story = { render: () => , tags: ['desktop-e2e'], - wdio: { - 'closes on Tab and continues past the anchor, and on Shift+Tab returns to it': async ({ browser, expect, platform, skip }) => { - const features = browser.capabilities['furn:features']; - if (platform === 'win32') { - skip('Win32 hosts keep the cycle behavior (#4377).'); - return; - } - if (!features?.physicalClick || !features.keyboard || !features.focus) { - skip('Tab exit requires physical owner input, keyboard input, and focus observation.'); - return; - } - const trigger = await browser.$('~modern-menu-trigger'); - const status = await browser.$('~modern-menu-status'); - await trigger.click(); - await expect(status).toHaveText('Ready: confirmed'); - await browser.keys('\uE004'); - await expect(status).toHaveText('Dismissed: tab-forward'); - await browser.waitUntil(async () => (await (await browser.$('~modern-menu-after')).getProperty('focused')) === true, { - timeout: 5000, - timeoutMsg: 'Expected Tab to continue to the view after the anchor.', - }); - await trigger.click(); - await expect(status).toHaveText('Ready: confirmed'); - await browser.keys(['\uE008', '\uE004']); - await expect(status).toHaveText('Dismissed: tab-backward'); - await browser.waitUntil(async () => (await trigger.getProperty('focused')) === true, { - timeout: 5000, - timeoutMsg: 'Expected Shift+Tab to focus the anchor.', - }); - }, - }, + wdio: [ + desktopIt( + 'closes on Tab and continues past the anchor, and on Shift+Tab returns to it', + { + requires: ['physical-click', 'keyboard', 'focus'], + unsupported: { platforms: ['win32'], reason: 'Win32 hosts keep the cycle behavior.', issue: '#4377' }, + }, + async ({ browser, expect }) => { + const trigger = await browser.$('~modern-menu-trigger'); + const status = await browser.$('~modern-menu-status'); + await trigger.click(); + await expect(status).toHaveText('Ready: confirmed'); + await browser.keys('\uE004'); + await expect(status).toHaveText('Dismissed: tab-forward'); + await browser.waitUntil(async () => (await (await browser.$('~modern-menu-after')).getProperty('focused')) === true, { + timeout: 5000, + timeoutMsg: 'Expected Tab to continue to the view after the anchor.', + }); + await trigger.click(); + await expect(status).toHaveText('Ready: confirmed'); + await browser.keys(['\uE008', '\uE004']); + await expect(status).toHaveText('Dismissed: tab-backward'); + await browser.waitUntil(async () => (await trigger.getProperty('focused')) === true, { + timeout: 5000, + timeoutMsg: 'Expected Shift+Tab to focus the anchor.', + }); + }, + ), + ], }; export const TabDismissal: Story = { render: () => , tags: ['desktop-e2e'], - wdio: { - 'closes on Tab and leaves the owner window its retained focus': async ({ browser, expect, platform, skip }) => { - const features = browser.capabilities['furn:features']; - if (platform === 'win32') { - skip('Win32 hosts keep the cycle behavior (#4377).'); - return; - } - if (!features?.physicalClick || !features.keyboard || !features.focus) { - skip('Tab exit requires physical owner input, keyboard input, and focus observation.'); - return; - } - const trigger = await browser.$('~modern-menu-trigger'); - const status = await browser.$('~modern-menu-status'); - await trigger.click(); - await expect(status).toHaveText('Ready: confirmed'); - await browser.keys('\uE004'); - await expect(status).toHaveText('Dismissed: tab-forward'); - await browser.waitUntil(async () => (await trigger.getProperty('focused')) === true, { - timeout: 5000, - timeoutMsg: 'Expected the owner window to keep the focused trigger.', - }); - expect(await (await browser.$('~modern-menu-after')).getProperty('focused')).toBe(false); - }, - }, + wdio: [ + desktopIt( + 'closes on Tab and leaves the owner window its retained focus', + { + requires: ['physical-click', 'keyboard', 'focus'], + unsupported: { platforms: ['win32'], reason: 'Win32 hosts keep the cycle behavior.', issue: '#4377' }, + }, + async ({ browser, expect }) => { + const trigger = await browser.$('~modern-menu-trigger'); + const status = await browser.$('~modern-menu-status'); + await trigger.click(); + await expect(status).toHaveText('Ready: confirmed'); + await browser.keys('\uE004'); + await expect(status).toHaveText('Dismissed: tab-forward'); + await browser.waitUntil(async () => (await trigger.getProperty('focused')) === true, { + timeout: 5000, + timeoutMsg: 'Expected the owner window to keep the focused trigger.', + }); + expect(await (await browser.$('~modern-menu-after')).getProperty('focused')).toBe(false); + }, + ), + ], }; export const MenuCompositionFocusingTrigger: Story = { render: ({ pointerFocus }) => , args: { pointerFocus: 'always' }, argTypes: MenuComposition.argTypes, tags: ['desktop-e2e'], - wdio: { - 'restores keyboard focus to a trigger focused on press': async ({ browser, expect, skip }) => { - const { exerciseMenuComposition } = await import('./menuComposition.wdio.ts'); - await exerciseMenuComposition({ browser, expect, skip }, { restoresTrigger: true }); - }, - }, + wdio: [ + desktopIt( + 'restores keyboard focus to a trigger focused on press', + { requires: ['physical-click', 'keyboard', 'focus'] }, + async ({ browser, expect }) => { + const { exerciseMenuComposition } = await import('./menuComposition.wdio.ts'); + await exerciseMenuComposition({ browser, expect }, { restoresTrigger: true }); + }, + ), + ], }; const styles = StyleSheet.create({ story: { padding: 24, gap: 16 }, diff --git a/packages/agentic/native-core/src/components/callout/menuComposition.wdio.ts b/packages/agentic/native-core/src/components/callout/menuComposition.wdio.ts index ecf57e52c0..38f8554eea 100644 --- a/packages/agentic/native-core/src/components/callout/menuComposition.wdio.ts +++ b/packages/agentic/native-core/src/components/callout/menuComposition.wdio.ts @@ -2,18 +2,14 @@ import type { WdioStoryContext } from '@fluentui-react-native/storybook-desktop/ /** * Opens the menu by pointer, enters its FocusZone, navigates by arrow with wrapping, and closes it with Escape, twice. - * `restoresTrigger` states whether the pressed trigger took keyboard focus, which decides whether the owner window - * returns focus to it when the menu closes. + * The calling case declares the physical click, keyboard and focus observation it needs. `restoresTrigger` states + * whether the pressed trigger took keyboard focus, which decides whether the owner window returns focus to it when the + * menu closes. */ export async function exerciseMenuComposition( - { browser, expect, skip }: Pick, + { browser, expect }: Pick, { restoresTrigger }: { restoresTrigger: boolean }, ): Promise { - const features = browser.capabilities['furn:features']; - if (!features?.physicalClick || !features.keyboard || !features.focus) { - skip('Menu composition requires physical owner input, keyboard input, and focus observation.'); - return; - } const trigger = await browser.$('~modern-menu-trigger'); const status = await browser.$('~modern-menu-status'); const focused = await browser.$('~modern-menu-focus'); diff --git a/packages/agentic/native-core/src/components/focus-zone/FocusZone.stories.tsx b/packages/agentic/native-core/src/components/focus-zone/FocusZone.stories.tsx index 9614a69b70..b9fd07ca76 100644 --- a/packages/agentic/native-core/src/components/focus-zone/FocusZone.stories.tsx +++ b/packages/agentic/native-core/src/components/focus-zone/FocusZone.stories.tsx @@ -2,6 +2,7 @@ import * as React from 'react'; import { Pressable, Text, View, StyleSheet } from 'react-native'; import type { Meta, StoryObj } from '@storybook/react-native'; +import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; import type { WdioStory } from '@fluentui-react-native/storybook-desktop/testing'; import { FocusZone } from './FocusZone'; @@ -150,71 +151,63 @@ export default meta; type Story = WdioStory>; export const Default: Story = { tags: ['desktop-e2e'], - wdio: { - 'confirms native default, edge, and specific descendant requests': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Native commands require physical input for their owner controls.'); - return; - } - for (const [strategy, item] of [ - ['default', 2], - ['first', 1], - ['last', 3], - ['default', 2], - ['target', 2], - ['first', 1], - ] as const) { - await (await browser.$(`~modern-focus-request-${strategy}`)).click(); - await expect(await browser.$('~modern-focus-status')).toHaveText('confirmed'); - await browser.waitUntil(async () => (await (await browser.$(`~modern-focus-item-${item}`)).getProperty('focused')) === true, { - timeout: 5000, - timeoutMsg: `Expected actual focus on item ${item}.`, - }); - } - }, - }, + wdio: [ + desktopIt( + 'confirms native default, edge, and specific descendant requests', + { requires: ['physical-click'] }, + async ({ browser, expect }) => { + for (const [strategy, item] of [ + ['default', 2], + ['first', 1], + ['last', 3], + ['default', 2], + ['target', 2], + ['first', 1], + ] as const) { + await (await browser.$(`~modern-focus-request-${strategy}`)).click(); + await expect(await browser.$('~modern-focus-status')).toHaveText('confirmed'); + await browser.waitUntil(async () => (await (await browser.$(`~modern-focus-item-${item}`)).getProperty('focused')) === true, { + timeout: 5000, + timeoutMsg: `Expected actual focus on item ${item}.`, + }); + } + }, + ), + ], }; export const Disabled: Story = { args: { disabled: true }, - wdio: { - 'refuses a disabled-zone request': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Physical input unavailable.'); - return; - } + wdio: [ + desktopIt('refuses a disabled-zone request', { requires: ['physical-click'] }, async ({ browser, expect }) => { await (await browser.$('~modern-focus-request-first')).click(); await expect(await browser.$('~modern-focus-status')).toHaveText('not-focusable'); - }, - }, + }), + ], }; export const TargetLifetimes: Story = { render: () => , - wdio: { - 'rejects unfocusable and outside live targets': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Physical input unavailable.'); - return; - } + wdio: [ + desktopIt('rejects unfocusable and outside live targets', { requires: ['physical-click'] }, async ({ browser, expect }) => { await (await browser.$('~modern-focus-request-unfocusable')).click(); await expect(await browser.$('~modern-focus-status')).toHaveText('not-focusable'); await (await browser.$('~modern-focus-request-outside')).click(); await expect(await browser.$('~modern-focus-status')).toHaveText('not-mounted'); expect(await (await browser.$('~modern-focus-outside')).getProperty('focused')).toBe(false); - }, - 'cancels pre-aborted requests and invalidates detached commands': async ({ browser, expect, skip }) => { - if (!browser.capabilities['furn:features']?.physicalClick) { - skip('Physical input unavailable.'); - return; - } - await (await browser.$('~modern-focus-request-cancelled')).click(); - await expect(await browser.$('~modern-focus-status')).toHaveText('cancelled'); - expect(await (await browser.$('~modern-focus-item-1')).getProperty('focused')).toBe(false); - await (await browser.$('~modern-focus-unmount')).click(); - await expect(await browser.$('~modern-focus-zone')).not.toExist(); - await (await browser.$('~modern-focus-request-detached')).click(); - await expect(await browser.$('~modern-focus-status')).toHaveText('not-mounted'); - }, - }, + }), + desktopIt( + 'cancels pre-aborted requests and invalidates detached commands', + { requires: ['physical-click'] }, + async ({ browser, expect }) => { + await (await browser.$('~modern-focus-request-cancelled')).click(); + await expect(await browser.$('~modern-focus-status')).toHaveText('cancelled'); + expect(await (await browser.$('~modern-focus-item-1')).getProperty('focused')).toBe(false); + await (await browser.$('~modern-focus-unmount')).click(); + await expect(await browser.$('~modern-focus-zone')).not.toExist(); + await (await browser.$('~modern-focus-request-detached')).click(); + await expect(await browser.$('~modern-focus-status')).toHaveText('not-mounted'); + }, + ), + ], }; const styles = StyleSheet.create({ story: { padding: 24, gap: 16 }, diff --git a/packages/agentic/native-core/src/legacy/callout/Callout.stories.tsx b/packages/agentic/native-core/src/legacy/callout/Callout.stories.tsx index 2a4d0b6e8b..864f43add9 100644 --- a/packages/agentic/native-core/src/legacy/callout/Callout.stories.tsx +++ b/packages/agentic/native-core/src/legacy/callout/Callout.stories.tsx @@ -2,6 +2,7 @@ import * as React from 'react'; import { Pressable, StyleSheet, Text, View } from 'react-native'; +import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; import type { WdioStory } from '@fluentui-react-native/storybook-desktop/testing'; import type { Meta, StoryObj } from '@storybook/react-native'; @@ -185,32 +186,31 @@ type Story = WdioStory>; export const Default: Story = { tags: ['desktop-e2e'], render: (args) => , - wdio: { - 'opens, light-dismisses, and recreates a native popup': async ({ browser, expect, platform, skip }) => { - const features = browser.capabilities['furn:features']; - if (!features?.physicalClick) { - skip('Popup lifecycle qualification requires physical pointer input.'); - return; - } - const status = await browser.$('~agentic-storybook-callout-status'); - for (let attempt = 0; attempt < 2; attempt++) { - await expect(status).toHaveText('Native window: Shown'); - // Only the AppKit provider enumerates the popup as a top-level window. - if (platform === 'macos') expect(await browser.getWindowHandles()).toHaveLength(2); - await status.click(); - if (platform === 'macos') { - await browser.waitUntil(async () => (await browser.getWindowHandles()).length === 1, { - timeout: 5000, - timeoutMsg: 'Expected dismissal to remove the native Callout window.', - }); - } - await expect(status).toHaveText('Native window: Dismissed'); - if (attempt === 0) { - await (await browser.$('~agentic-storybook-callout-trigger')).click(); + wdio: [ + desktopIt( + 'opens, light-dismisses, and recreates a native popup', + { requires: ['physical-click'] }, + async ({ browser, expect, platform }) => { + const status = await browser.$('~agentic-storybook-callout-status'); + for (let attempt = 0; attempt < 2; attempt++) { + await expect(status).toHaveText('Native window: Shown'); + // Only the AppKit provider enumerates the popup as a top-level window. + if (platform === 'macos') expect(await browser.getWindowHandles()).toHaveLength(2); + await status.click(); + if (platform === 'macos') { + await browser.waitUntil(async () => (await browser.getWindowHandles()).length === 1, { + timeout: 5000, + timeoutMsg: 'Expected dismissal to remove the native Callout window.', + }); + } + await expect(status).toHaveText('Native window: Dismissed'); + if (attempt === 0) { + await (await browser.$('~agentic-storybook-callout-trigger')).click(); + } } - } - }, - }, + }, + ), + ], }; export const Placement: Story = { @@ -230,19 +230,18 @@ export const WindowCommands: Story = { }, render: (args) => , tags: ['desktop-e2e'], - wdio: { - 'dismisses initially focused content on Escape': async ({ browser, expect, skip }) => { - const features = browser.capabilities['furn:features']; - if (!features?.physicalClick || !features.keyboard) { - skip('Escape dismissal requires an interactive desktop with keyboard input.'); - return; - } - const status = await browser.$('~agentic-storybook-callout-status'); - await expect(status).toHaveText('Native window: Shown'); - await browser.keys('\uE00C'); - await expect(status).toHaveText('Native window: Dismissed'); - }, - }, + wdio: [ + desktopIt( + 'dismisses initially focused content on Escape', + { requires: ['physical-click', 'keyboard'] }, + async ({ browser, expect }) => { + const status = await browser.$('~agentic-storybook-callout-status'); + await expect(status).toHaveText('Native window: Shown'); + await browser.keys('\uE00C'); + await expect(status).toHaveText('Native window: Dismissed'); + }, + ), + ], parameters: { docs: { description: { diff --git a/packages/agentic/storybook-desktop/config/server.cjs b/packages/agentic/storybook-desktop/config/server.cjs index 99421e224d..f4c461e76b 100644 --- a/packages/agentic/storybook-desktop/config/server.cjs +++ b/packages/agentic/storybook-desktop/config/server.cjs @@ -113,6 +113,11 @@ async function startDesktopStorybookServer({ keepAlive = setInterval(() => undefined, 60_000); if (driverManifest) { const { StorybookChannelOrchestrator } = await import('../lib/driver/index.js'); + // An accessible driver cannot perform physical input, so every test that attaches to it runs the accessible lane. + const maskedFeatures = + driverManifest.wdio?.input === 'accessible' + ? (await import('@fluentui-react-native/desktop-driver/authoring')).accessibleLaneMaskedFeatures + : undefined; const target = { endpoint: driverManifest.endpoint, id: driverManifest.targetId, @@ -130,6 +135,7 @@ async function startDesktopStorybookServer({ platformName: target.platformName, storyRootTestId: target.storyRootTestId, windows: createFakeStoryWindows(driverManifest.storyManifest, target.storyRootTestId), + ...(maskedFeatures ? { features: Object.fromEntries(maskedFeatures.map((feature) => [feature, false])) } : {}), }); driver = await createDesktopDriverServer({ host, @@ -149,6 +155,7 @@ async function startDesktopStorybookServer({ onStderr: (text) => report.reporter.emit({ kind: 'output', payload: { stream: 'stderr', text } }, 'native-helper', { service: 'native-helper' }), onFailure: fatal, + ...(maskedFeatures ? { disabledInputFeatures: maskedFeatures } : {}), ...(agent ? { agent: { descriptor: agent.descriptor, ...(agent.root === undefined ? {} : { root: agent.root }) }, diff --git a/packages/agentic/storybook-desktop/config/storybook-control.cjs b/packages/agentic/storybook-desktop/config/storybook-control.cjs index 40df0e989b..2e1f1aa09c 100644 --- a/packages/agentic/storybook-desktop/config/storybook-control.cjs +++ b/packages/agentic/storybook-desktop/config/storybook-control.cjs @@ -145,8 +145,12 @@ async function runAuthoredTests() { const configUrl = pathToFileURL(path.join(__dirname, '..', 'lib', 'config', 'index.js')).href; const { formatDesktopStorybookSmokeTestSummary, runDesktopStorybookSmokeTests } = await import(smokeTestsUrl); const { makeDesktopStorybookConfig } = await import(configUrl); + // The driver instance's lane is a per-run choice the CLI resolved, so it travels as an option, not as configuration. + const { input, ...wdio } = manifest.wdio ?? {}; const result = await runDesktopStorybookSmokeTests({ - config: makeDesktopStorybookConfig({ projectRoot: process.cwd(), wdio: manifest.wdio }), + config: makeDesktopStorybookConfig({ projectRoot: process.cwd(), wdio }), + ...(input ? { input } : {}), + ...(manifest.scope ? { scope: manifest.scope } : {}), // eslint-disable-next-line @microsoft/sdl/no-insecure-url -- the Desktop Driver is loopback-only driverUrl: `http://127.0.0.1:${manifest.driverPort}`, manifest: manifest.storyManifest, diff --git a/packages/agentic/storybook-desktop/config/transform-story-tests.cjs b/packages/agentic/storybook-desktop/config/transform-story-tests.cjs index bc47137db0..ae64781038 100644 --- a/packages/agentic/storybook-desktop/config/transform-story-tests.cjs +++ b/packages/agentic/storybook-desktop/config/transform-story-tests.cjs @@ -242,6 +242,136 @@ function collectWdioTests(program) { return tests; } +// The physical requirement each WebDriver action source type implies, in the order messages list them. +const actionSourceRequirements = { key: 'keyboard', pointer: 'physical-click', wheel: 'wheel' }; +const requirementInputKinds = { keyboard: 'keyboard', 'physical-click': 'pointer', wheel: 'wheel' }; +const valueCommands = new Set(['setValue', 'addValue', 'clearValue']); + +function memberName(callee) { + const property = callee.node.property; + if (!callee.node.computed) return types.isIdentifier(property) ? property.name : undefined; + if (types.isStringLiteral(property)) return property.value; + return types.isTemplateLiteral(property) && property.expressions.length === 0 ? property.quasis[0].value.cooked : undefined; +} + +function isBrowser(object) { + object = unwrap(object); + if (object.isIdentifier({ name: 'browser' })) return true; + return (object.isMemberExpression() || object.isOptionalMemberExpression()) && memberName(object) === 'browser'; +} + +/** Follows a constant to the value it holds, so a payload declared above the call is still read. */ +function readConstant(value, seen = new Set()) { + value = unwrap(value); + if (value.isIdentifier()) { + const binding = value.scope.getBinding(value.node.name); + const declarator = binding?.path; + if ( + binding?.constant && + declarator?.isVariableDeclarator() && + declarator.node.init && + declarator.get('id').isIdentifier() && + !seen.has(binding) + ) { + seen.add(binding); + return readConstant(declarator.get('init'), seen); + } + } + return value; +} + +function readString(value) { + value = readConstant(value); + if (value.isStringLiteral()) return value.node.value; + return value.isTemplateLiteral() && value.node.expressions.length === 0 ? value.node.quasis[0].value.cooked : undefined; +} + +function actionSourceType(source) { + source = readConstant(source); + if (!source.isObjectExpression()) return undefined; + let type; + for (const property of source.get('properties')) { + // A spread or a computed key could supply or replace the type. + if (!property.isObjectProperty() || property.node.computed) return undefined; + if (keyName(property) === 'type') type = readString(property.get('value')); + } + return type; +} + +/** The physical requirements of a WebDriver action payload, and whether every source in it could be read. */ +function readActionSources(payload) { + payload = readConstant(payload); + if (!payload.isArrayExpression()) return undefined; + const requires = new Set(); + let readable = true; + for (const element of payload.get('elements')) { + const type = element?.node && !element.isSpreadElement() ? actionSourceType(element) : undefined; + if (type === undefined) readable = false; + else if (Object.hasOwn(actionSourceRequirements, type)) requires.add(actionSourceRequirements[type]); + } + return { requires: Object.values(actionSourceRequirements).filter((requirement) => requires.has(requirement)), readable }; +} + +/** A call that sends physical input, with the requirements that input implies. Those are undefined when the call cannot be read. */ +function directInputCall(call) { + const callee = call.get('callee'); + if (!callee.isMemberExpression() && !callee.isOptionalMemberExpression()) return undefined; + const name = memberName(callee); + const argument = call.node.arguments[0] ? call.get('arguments.0') : undefined; + if (valueCommands.has(name)) return { call: name, requires: ['keyboard'] }; + if (name === 'keys' && isBrowser(callee.get('object'))) return { call: 'browser.keys', requires: ['keyboard'] }; + if (name === 'action' && isBrowser(callee.get('object'))) { + const type = argument && readString(argument); + if (type === undefined) return { call: 'browser.action', requires: undefined }; + return Object.hasOwn(actionSourceRequirements, type) + ? { call: 'browser.action', requires: [actionSourceRequirements[type]] } + : undefined; + } + if (name === 'performActions') { + const sources = argument && readActionSources(argument); + if (!sources) return { call: 'performActions', requires: undefined }; + if (sources.requires.length) return { call: 'performActions', requires: sources.requires }; + return sources.readable ? undefined : { call: 'performActions', requires: undefined }; + } + return undefined; +} + +/** + * Rejects a case that also runs in the accessible input lane but sends physical input itself: the keyboard commands, the value + * commands that may type, and WebDriver actions with key, pointer or wheel sources. The check reads literals only. Input sent + * through a helper or an alias is stopped by the accessible lane's host masks when the case runs. + */ +function rejectDirectInput(callback, { name, input, inputSource }) { + if (input === 'physical') return; + let finding; + callback.traverse({ + 'CallExpression|OptionalCallExpression'(call) { + const found = finding ? undefined : directInputCall(call); + if (found) finding = { ...found, path: call }; + }, + }); + if (!finding) return; + const subject = name ? `"${name}"` : 'This case'; + const declared = inputSource === 'declared'; + if (finding.requires === undefined) { + const declaration = declared ? "input: 'physical'" : 'requires for the physical input it sends (keyboard, physical-click or wheel)'; + fail( + finding.path, + `${subject} runs in the accessible input lane, but calls ${finding.call} with an argument this check cannot read. ` + + `Pass a literal list of input sources, or declare ${declaration}.`, + ); + } + const kinds = finding.requires.map((requirement) => requirementInputKinds[requirement]).join(' and '); + const declaration = declared + ? "input: 'physical'" + : `requires: [${finding.requires.map((requirement) => `'${requirement}'`).join(', ')}]`; + fail( + finding.path, + `${subject} runs in the accessible input lane, but ${finding.call} sends physical ${kinds} input. ` + + `Declare ${declaration} so it runs only in the physical lane, or remove the call.`, + ); +} + function compileWdioTest(callback, sourceFile, source) { const file = types.file(types.program([types.expressionStatement(types.cloneNode(callback.node, true))])); const result = transformFromAstSync(file, source, { @@ -365,3 +495,4 @@ module.exports = () => ({ module.exports.collectWdioTests = collectWdioTests; module.exports.compileWdioTest = compileWdioTest; +module.exports.rejectDirectInput = rejectDirectInput; diff --git a/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts b/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts index 67e6ed935c..e620c955b8 100644 --- a/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts +++ b/packages/agentic/storybook-desktop/config/transform-story-tests.d.cts @@ -16,6 +16,10 @@ declare namespace storyTests { function collectWdioTests(program: NodePath): ReadonlyMap; function compileWdioTest(callback: Callback['callback'], sourceFile: string, source: string): string; + function rejectDirectInput( + callback: Callback['callback'], + descriptor: { name?: string; input: 'accessible' | 'physical' | 'both'; inputSource: 'declared' | 'derived' }, + ): void; } export = storyTests; diff --git a/packages/agentic/storybook-desktop/references/commands.md b/packages/agentic/storybook-desktop/references/commands.md index f60208c21c..a82fa228d0 100644 --- a/packages/agentic/storybook-desktop/references/commands.md +++ b/packages/agentic/storybook-desktop/references/commands.md @@ -29,6 +29,8 @@ only). The mode precedence is CLI flag, `DESKTOP_REPORTER`, config, then | `test -- --list` | Discover authored executable story-test IDs | | `test -- --test ` | Run matching isolated story callbacks | | `test -- --input accessible` | Select accessible and `both` descriptor cases | +| `test -- --list --input physical` | Preview a lane's selection and exclusions | +| `driver -- --input accessible` | Start a driver that masks physical input | | `smoke -- --mode stories` | Build/bundle/launch, traverse stories, clean up | | `smoke -- --mode stories-and-tests` | Traverse stories and execute callbacks | @@ -63,10 +65,16 @@ Windows smokes. Storybook test and stories-and-tests smoke commands accept `--input accessible|physical`. The lane filters `desktopIt` cases by their -resolved input and target platform. `driver --input` records the default lane -for subsequent tests against that instance. Legacy callback collections are -not eligible for input-lane runs. Input lanes cannot be combined with an -explicit non-`auto` click mode. +resolved input, target platform, and scope. Without a flag the lane comes from +`DESKTOP_INPUT`, then from the driver instance a `test` attaches to; none of them +means compatibility mode, which runs every case as before. `driver --input` +starts the driver for that lane and records it for subsequent tests against the +instance. An `accessible` driver masks keyboard, physical click, and wheel, and +`test --input accessible` verifies the mask before running anything: a driver +that is not masked fails the run rather than letting a case move the pointer. +Legacy callback collections are not eligible for input-lane runs. Input lanes +cannot be combined with an explicit non-`auto` click mode. See +[story tests](story-tests.md#run-an-input-lane). The TypeScript API exposes the same operations through `DesktopStorybookCli`. Higher-level supervisors should inject their shared reporter, runner, and diff --git a/packages/agentic/storybook-desktop/references/contributing.md b/packages/agentic/storybook-desktop/references/contributing.md index 207be56c30..8a1ac7efea 100644 --- a/packages/agentic/storybook-desktop/references/contributing.md +++ b/packages/agentic/storybook-desktop/references/contributing.md @@ -17,6 +17,16 @@ The channel child in `config/server.cjs` constructs its native driver through helper policy, authority, descriptor publication and settled driver shutdown in that shared harness implementation, not a parallel Storybook copy. +Input lanes cross these areas. The descriptor schema, merge, and per-case +decision belong to `@fluentui-react-native/desktop-driver/authoring`; do not +copy them here. `src/cli/inputLane.ts` resolves the requested lane, +`src/testing/selection.ts` decides every case before launch and renders +`test --list`, `src/testing/laneVerdict.ts` judges a lane run's accounting, and +`config/server.cjs` masks the accessible lane's physical input on the driver. +`config/transform-story-tests.cjs` is both the Metro Babel plugin and the shared +AST library: its `rejectDirectInput` check runs from `src/testing/extractWdioTests.ts` +only, because the Metro transform has no validated descriptor to judge by. + Keep story callbacks statically extractable and authentication checks fail-closed. Update references when commands, supported config, outputs, or readiness contracts change. diff --git a/packages/agentic/storybook-desktop/references/story-tests.md b/packages/agentic/storybook-desktop/references/story-tests.md index 62662edbbd..438c0e09a7 100644 --- a/packages/agentic/storybook-desktop/references/story-tests.md +++ b/packages/agentic/storybook-desktop/references/story-tests.md @@ -7,63 +7,123 @@ owner_paths: # Author story tests -Story tests are statically extracted CSF `wdio` callbacks. Use `desktopIt` -descriptors when declaring input requirements or endpoint scope. Top-level -functions or named callback collections remain supported for compatibility, -but cannot be selected by an input lane. Callbacks must not close over -module-local values. Import Node-only helpers through literal dynamic -`import()` inside the callback. Keep test code out of the native bundle. +Story tests are statically extracted CSF `wdio` callbacks. Declare each case +with `desktopIt`: its descriptor says which input the case needs and where it +runs, so a run can select it for the input lane, platform, and scope it +exercises. Callbacks must not close over module-local values. Import Node-only +helpers through literal dynamic `import()` inside the callback. Keep test code +out of the native bundle. ```tsx import type { StoryObj } from '@storybook/react-native'; +import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; import type { WdioStory } from '@fluentui-react-native/storybook-desktop/testing'; type Story = WdioStory>; export const Default: Story = { - args: { testID: 'save-button' }, - wdio: { - 'is enabled': async ({ browser, expect }) => { - await expect(await browser.$('~save-button')).toBeEnabled(); - }, - 'supports activation': async ({ browser, platform, skip }) => { - const features = browser.capabilities['furn:features']; - if (!features?.physicalClick) { - skip('Physical pointer input is unavailable.'); - return; - } - await (await browser.$('~save-button')).click(); - }, - }, -}; -``` - -For modality-aware tests, import the value helper and use an array of cases: - -```tsx -import { desktopIt, type WdioStory } from '@fluentui-react-native/storybook-desktop/testing'; - -export const Default: WdioStory> = { args: { testID: 'save-button' }, wdio: [ - desktopIt('activates in either input lane', { requires: ['click'] }, async ({ browser, expect }) => { + desktopIt('is enabled', async ({ browser, expect }) => { + await expect(await browser.$('~save-button')).toBeEnabled(); + }), + desktopIt('activates', { requires: ['click'] }, async ({ browser, expect }) => { await (await browser.$('~save-button')).click(); await expect(await browser.$('~saved-message')).toBeDisplayed(); }), - desktopIt('uses a physical pointer', { requires: ['physical-click'], platforms: ['win32'] }, async ({ browser }) => { - await (await browser.$('~save-button')).click(); + desktopIt('activates with Enter', { requires: ['keyboard'], platforms: ['!macos'] }, async ({ browser, expect }) => { + await browser.keys('\uE007'); + await expect(await browser.$('~saved-message')).toBeDisplayed(); }), ], }; ``` -`requires: ['click']` derives to `both` and adapts `element.click()` to the -selected lane: accessibility click in the accessible lane and physical click -in the physical lane. Explicit physical or accessibility requirements derive -their matching lane. `input` may narrow a `both` case but cannot widen a -transport-specific requirement. `platforms` is an allowlist by default; -`['!win32']` excludes that endpoint. Platform selection uses the target -endpoint, so Win32 remains distinct from Windows. +`desktopIt(name, run)` declares a case with no restrictions, so it runs in every +lane on every platform. `desktopIt(name, options, run)` adds a descriptor. The +older object of named callbacks and the single `wdio: async (context) => ...` +form remain supported for compatibility, but a legacy callback has no +descriptor, so an input lane rejects it, listing the story IDs to migrate, +instead of treating it as `both`. + +## Declare how a case runs + +The descriptor is a strict static JSON literal; an unknown field, a computed +value, or an invalid combination fails extraction with source context. + +| Field | Meaning | +| ------------- | ---------------------------------------------------------------------------------------------------- | +| `requires` | Host capabilities the case needs. They also derive the input lane (below). | +| `input` | `accessible`, `physical`, or `both`. Defaults to what `requires` derives, else `both`. | +| `platforms` | Endpoints the case runs on: `macos`, `windows`, `win32`. Omitted means every endpoint. | +| `scope` | Custom scope values, such as `{ 'rn.renderer': ['fabric'] }`. Omitted means every value. | +| `unsupported` | A known gap: `{ platforms, reason, issue? }`. The case is reported unsupported instead of executing. | +| `quarantine` | A temporary gap: `{ issue, reason, expires, platforms?, input? }`, with `expires` as `YYYY-MM-DD`. | + +### Input and requirements + +Most cases need no `input`: their `requires` already says what they send. + +| Requirement | Derived input | Meaning | +| --------------------------------------------------------------------------------------- | ------------- | ---------------------------------------------------------------------- | +| `keyboard`, `physical-click`, `wheel` | `physical` | Operating-system input that moves the real pointer or sends key events | +| `accessibility-click` | `accessible` | The element's accessibility action | +| `click` | `both` | `element.click()`, which follows the lane (below) | +| `accessibility-actions`, `element-screenshot`, `focus`, `screenshot`, `set-window-rect` | `both` | The host must offer the feature | + +A case with no requirements is `both` and runs in every lane. Requiring both a +physical and an accessibility transport is an error; split the case. An explicit +`input` may narrow a `both` case but cannot widen a transport requirement. + +`requires: ['click']` makes `element.click()` follow the lane: an accessibility +click in the accessible lane and a physical click in the physical lane. Without +a lane, an accessible case still uses accessibility clicks and every other case +follows the configured `--click-mode`, so a physical case on a host that masks +physical click is reported unsupported instead of failing its session. Before +a case runs, the host's reported features must meet each requirement. A case it +cannot meet is reported unsupported, never passed, and makes a lane run +incomplete unless the case declares the gap. + +### Platforms and scope + +`platforms` is an allowlist: `['win32']` runs only on Win32. Negate with +`['!win32']` to run everywhere else; a list holds either allowed or negated +endpoints, never both. Platform selection uses the target endpoint, so Win32 +remains distinct from Windows. + +`scope` narrows by values the target holds. Values within a key are alternatives +and keys combine. `rn.renderer` (`fabric` or `paper`) is built in and the +`rn.*` keys are reserved for the framework. A consumer declares its own keys in +the desktop config's `testScopes`, such as `{ flavor: ['stable', 'preview'] }`, +and each target sets its value in `testScope`. A key or value outside the declared +domains, or a key the target does not supply, is a configuration error rather +than a silent exclusion, so a typo cannot drop a case. + +A case that does not match the target is excluded, not skipped: it is not a test +and is counted only in the run's selection totals. + +### Known gaps + +Declare a gap instead of calling `skip` at run time. `unsupported` lists the +endpoints where the behavior does not work yet, with a reason and ideally an +issue. `quarantine` suspends a flaky case until its `expires` date, optionally for +one endpoint or lane. An expired quarantine fails the run before anything +executes. Declared gaps are reported as unsupported with their reason and do not +make a lane run incomplete; a run-time `skip(reason)` does, so reserve it for +conditions no descriptor can state. + +```tsx +desktopIt( + 'closes on Tab and continues past the anchor', + { + requires: ['physical-click', 'keyboard', 'focus'], + unsupported: { platforms: ['win32'], reason: 'Win32 hosts keep the cycle behavior.', issue: '#4377' }, + }, + async ({ browser, expect }) => { + // ... + }, +); +``` The callback receives the real WebdriverIO `browser`, `expect-webdriverio` `expect`, typed `desktop` assertions/story commands, target `platform` (`macos`, @@ -85,10 +145,10 @@ Metro resolves their dependencies. Use native selectors and WebDriver operations; the app has no DOM or JavaScript execution surface. Read native properties through `getProperty()` rather than -ordinary object matchers. Feature-check native actions and explicitly skip -unsupported cases; never turn unsupported state into false. For example, -`getProperty('accessibilityActions')` lists the action names an element -publishes (such as `Toggle` or `Select`) when +ordinary object matchers. Declare the capabilities a case needs in `requires` +rather than feature-checking inside the callback, and never turn unsupported +state into false. For example, `getProperty('accessibilityActions')` lists the +action names an element publishes (such as `Toggle` or `Select`) when `browser.capabilities['furn:features']?.accessibilityActions` is true. Tests in one story run in declaration order, each in its own Node process and @@ -101,12 +161,62 @@ editing callbacks because their digest participates in manifest freshness. filters names. `test` operates against an already running app/driver; smoke owns the complete app lifecycle. +## Run an input lane + +A lane runs only the cases that apply to one way of driving the app. The +`accessible` lane runs `accessible` and `both` cases through accessibility +actions, so it never moves the pointer or sends key events and is safe on a +machine someone is using. The `physical` lane runs `physical` and `both` cases +with real pointer, keyboard, and wheel input, so it needs a machine nobody is +using. Without a lane, every case runs as before (compatibility mode) and click +behavior follows `--click-mode`. + Use `storybook test --input accessible|physical` or -`storybook smoke --mode stories-and-tests --input accessible|physical` to -select declared cases. `storybook driver --input ...` supplies the default -lane to a later `storybook test` invocation. `--input` cannot be combined -with a non-default `--click-mode`. A lane requires descriptor cases and rejects -legacy callback collections rather than silently treating them as `both`. +`storybook smoke --mode stories-and-tests --input accessible|physical` to run +one lane. `DESKTOP_INPUT` supplies the lane when no flag is given. +`storybook driver --input ...` records the default lane for a later +`storybook test` against that driver. `--input` cannot be combined with a +non-default `--click-mode`. `test --list` shows, for each story, the cases the +lane runs with their resolved input, requirements, platforms, and scope, and the +cases it excludes with the reason; add `--input` to preview a lane. + +The accessible lane holds a case to accessible input by masking the host, not by +trusting the case: the driver reports keyboard, physical click, and wheel as +unavailable, so a case that tries them fails instead of silently moving the +pointer. `smoke` and `driver --input accessible` start the driver already +masked. `test --input accessible` attaches to a running driver and verifies the +mask from the session's reported features before it runs anything. A driver that +does not report all three as unavailable stops the run as an infrastructure +failure rather than running unmasked, so attach the lane to a driver started +with `driver --input accessible`. + +A lane that executes no case is not a passing run, and neither is a lane run +that leaves a case unsupported or skipped without a declared gap. Excluded cases +never count against a run: they are not tests. + +## Keep physical input out of accessible cases + +Extraction checks every `desktopIt` case that can run in the accessible lane +(any case that is not `physical`) and fails with a source-context error when its +callback calls an API that sends physical input without declaring it: + +- `browser.keys(...)`, which sends key events (`keyboard`); +- `setValue`, `addValue`, or `clearValue` on any object (`keyboard`); +- `browser.action('key' | 'pointer' | 'wheel')` and `performActions` with `key`, + `pointer`, or `wheel` sources (`keyboard`, `physical-click`, or `wheel`). + +Declare the matching `requires`, or `input: 'physical'` when the case already +declares its input, so it runs only in the physical lane, or remove the call. A +call whose arguments the check cannot read, such as a computed action list, +fails the same way, because it cannot tell what the call sends. The message +names the call and the requirement to declare. + +The check reads the callback only. It does not follow helper functions or aliases, +and `setValue`, `addValue`, and `clearValue` are matched by name, so a method you +define with one of those names is flagged too. Input hidden behind a helper is +instead caught at run time by the accessible lane's host mask. Legacy callbacks +are not checked, and the Metro transform does not run the check; extraction +(`storybook manifest`, `test`, `smoke`) does. Use the component authoring contract for component-specific test assertions. Legacy `parameters.desktopDriver` static plans remain for compatibility and diff --git a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts index c94989b0b2..8729f91a2c 100644 --- a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts +++ b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.test.ts @@ -2,7 +2,8 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import type { NativeDriverArtifact } from '@fluentui-react-native/desktop-driver'; +import type { DesktopStoryManifestEntry, NativeDriverArtifact } from '@fluentui-react-native/desktop-driver'; +import { validateDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; import { DesktopServiceScope } from '@fluentui-react-native/desktop-test'; import type { ResolvedDesktopConfig, ResolvedDesktopTarget } from '@fluentui-react-native/desktop-test'; import { selectDesktopRenderer } from '@fluentui-react-native/desktop-test/config'; @@ -11,6 +12,7 @@ import { STORYBOOK_SMOKE_MODE } from '../config/commands'; import { DesktopStorybookConfig, makeDesktopStorybookConfig } from '../config/makeDesktopStorybookConfig'; import { FURN_STORYBOOK_BUNDLE_IDENTIFIER, FURN_STORYBOOK_INSTANCE_ID } from '../config/instance'; import { FURN_STORYBOOK_PLATFORM } from '../config/platforms'; +import { desktopStorybookTargetId } from '../driver/driverManifest'; import type { DesktopCommandRunner, PreparedDesktopCommand, RunningDesktopCommand } from './commandRunner'; import { createDesktopStorybookCommand } from './createDesktopStorybookCommand'; import { @@ -76,6 +78,33 @@ const createEmptyStoryManifest = async (_config: unknown, platform: 'macos' | 'w schemaVersion: 1 as const, }); +function declaredStoryEntry(id: string, cases: readonly (readonly [name: string, options: unknown])[]): DesktopStoryManifestEntry { + return { + id, + name: 'Default', + title: 'Button', + sourcePath: 'button.stories.tsx', + packageName: 'fixture', + tags: [], + wdio: { + exportName: 'Default', + digest: 'declared', + testNames: cases.map(([name]) => name), + cases: cases.map(([name, options]) => ({ name, options: validateDesktopCaseOptions(options) })), + }, + }; +} + +function storyManifestOf(...entries: DesktopStoryManifestEntry[]) { + return async () => ({ + endpoint: 'macos' as const, + schemaVersion: 1 as const, + platformManifestDigest: 'digest', + portablePlanDigest: 'portable', + entries, + }); +} + describe('macOS application leases', () => { test('validates exact running application records and rejects ambiguity', () => { expect( @@ -360,6 +389,115 @@ describe('DesktopStorybookCli', () => { await expect(cli.test('macos', { timeoutMs: 0 })).rejects.toThrow('wdio.timeoutMs'); }); + describe('input lane selection', () => { + const cases = [ + ['adaptive', { requires: ['click'] }], + ['typing', { requires: ['keyboard'] }], + ['win32-only', { platforms: ['win32'] }], + ['paper-only', { scope: { 'rn.renderer': ['paper'] } }], + ['fabric-only', { scope: { 'rn.renderer': ['fabric'] } }], + ] as const; + const config = () => makeDesktopStorybookConfig({ projectRoot: storybookRoot, storyPackages: [] }); + const list = async (flags: string[], ...entries: DesktopStoryManifestEntry[]) => { + const output = { write: jest.fn() }; + await createDesktopStorybookCommand({ config: config(), createStoryManifest: storyManifestOf(...entries), output }).parseAsync([ + 'node', + 'storybook', + 'test', + '--macos', + '--list', + ...flags, + ]); + return JSON.parse(output.write.mock.calls[0][0]); + }; + // An explicit URL and target stand in for a running driver, so selection is what these runs exercise. + const connection = { url: 'https://127.0.0.1:1234', target: 'fixture' }; + const run = (options: Parameters[1], ...entries: DesktopStoryManifestEntry[]) => + new DesktopStorybookCli(config(), { createStoryManifest: storyManifestOf(...entries), output: { write: jest.fn() } }).test('macos', { + ...connection, + ...options, + }); + const legacyEntry = (id: string): DesktopStoryManifestEntry => ({ + ...declaredStoryEntry(id, []), + wdio: { exportName: 'Default', digest: 'legacy' }, + }); + + test.each([ + [ + 'accessible', + ['adaptive', 'fabric-only'], + [ + ['typing', 'input'], + ['win32-only', 'platform'], + ['paper-only', 'scope'], + ], + ], + [ + 'physical', + ['adaptive', 'typing', 'fabric-only'], + [ + ['win32-only', 'platform'], + ['paper-only', 'scope'], + ], + ], + ])('lists what the %s lane runs and why each other matching case is left out', async (input, selected, excluded) => { + const [story] = await list(['--input', input], declaredStoryEntry('button--default', cases)); + expect(story.tests.map(({ name }: { name: string }) => name)).toEqual(selected); + expect(story.excluded.map(({ name, reason }: { name: string; reason: string }) => [name, reason])).toEqual(excluded); + expect(story.tests[0]).toEqual({ name: 'adaptive', input: 'both', inputSource: 'derived', requires: ['click'] }); + }); + + test('lists every declared case that targets the platform when no lane is selected', async () => { + const [story] = await list([], declaredStoryEntry('button--default', cases)); + expect(story.tests.map(({ name }: { name: string }) => name)).toEqual(['adaptive', 'typing', 'fabric-only']); + expect(story.excluded.map(({ reason }: { reason: string }) => reason)).toEqual(['platform', 'scope']); + }); + + test('names every legacy callback a lane cannot run, so they migrate in one pass', async () => { + await expect(run({ input: 'accessible' }, legacyEntry('a--default'), legacyEntry('b--default'))).rejects.toThrow( + 'Input lane "accessible" cannot run legacy wdio callbacks; declare them with desktopIt: a--default, b--default.', + ); + }); + + test('explains an empty selection by what the target and lane excluded', async () => { + await expect(run({ input: 'accessible' }, declaredStoryEntry('button--default', cases.slice(1, 3)))).rejects.toThrow( + 'No executable wdio story tests matched the requested selection (excluded 1 by platform, 0 by scope, 1 by input).', + ); + }); + + test('does not blame a narrow filter for cases it never asked for', async () => { + await expect(run({ input: 'accessible', test: 'typing' }, declaredStoryEntry('button--default', cases))).rejects.toThrow( + 'No executable wdio story tests matched the requested selection (excluded 0 by platform, 0 by scope, 1 by input).', + ); + }); + + test('takes the lane of the running driver instance unless a flag or the environment names one', async () => { + const manifestPath = path.join(storybookRoot, 'storybook-desktop.generated', 'driver-manifest.macos.json'); + const cli = new DesktopStorybookCli(config(), { + createStoryManifest: storyManifestOf(declaredStoryEntry('button--default', cases.slice(1, 2))), + output: { write: jest.fn() }, + }); + fs.mkdirSync(path.dirname(manifestPath), { recursive: true }); + fs.writeFileSync( + manifestPath, + JSON.stringify({ + endpoint: 'macos', + instanceId: cli.instance.id, + renderer: 'fabric', + driverPort: 7100, + targetId: desktopStorybookTargetId(config(), 'macos', 'fabric'), + wdio: { input: 'accessible' }, + }), + ); + try { + // The only case needs a physical keyboard, so only the inherited accessible lane can leave it out. + await expect(cli.test('macos', {})).rejects.toThrow('excluded 0 by platform, 0 by scope, 1 by input'); + } finally { + fs.rmSync(manifestPath, { force: true }); + } + }); + }); + test('starts the config-owned server with platform and connection options', async () => { const runner = new RecordingRunner(); const services = new DesktopServiceScope(runner); @@ -486,6 +624,9 @@ describe('DesktopStorybookCli', () => { expect(fabricManifest.renderer).toBe('fabric'); expect(paperManifest.renderer).toBe('paper'); + // The control process that runs smoke tests has no desktop configuration, so the manifest carries the scope values. + expect(fabricManifest.scope.values).toMatchObject({ 'rn.renderer': 'fabric' }); + expect(paperManifest.scope.values).toMatchObject({ 'rn.renderer': 'paper' }); expect(paperManifest.bridgeNonce).not.toBe(fabricManifest.bridgeNonce); expect(repeatedPaperManifest.bridgeNonce).toBe(paperManifest.bridgeNonce); } finally { @@ -1018,6 +1159,7 @@ describe('DesktopStorybookCli', () => { manifest: await createEmptyStoryManifest(cli.config, 'macos'), platform: 'macos', report: undefined, + scope: { domains: { 'rn.renderer': ['fabric', 'paper'] }, values: { 'rn.renderer': 'fabric' } }, signal: expect.any(AbortSignal), targetId: 'agenticstorybook-macos', }); diff --git a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts index ad62fc01bf..ff7da2de2f 100644 --- a/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts +++ b/packages/agentic/storybook-desktop/src/cli/DesktopStorybookCli.ts @@ -2,8 +2,14 @@ import { spawn } from 'node:child_process'; import fs from 'node:fs'; import { createServer } from 'node:net'; import path from 'node:path'; -import { defaultAgentDescriptor, DesktopServiceScope, resolveDesktopApplication } from '@fluentui-react-native/desktop-test'; +import { + defaultAgentDescriptor, + DesktopServiceScope, + resolveDesktopApplication, + resolveDesktopTestScope, +} from '@fluentui-react-native/desktop-test'; import type { DesktopReportWriter } from '@fluentui-react-native/desktop-test/reporting'; +import { resolveDesktopScopeDomains } from '@fluentui-react-native/desktop-driver/authoring'; import { formatDesktopReporterValue } from '@fluentui-react-native/desktop-driver/cli/reporter'; import type { ResolvedDesktopReporterMode } from '@fluentui-react-native/desktop-driver/cli/reporter'; @@ -38,7 +44,9 @@ import { writeDesktopStoryManifest, } from '../driver/index.js'; import { + desktopStorybookDriverManifestPath, desktopStorybookTargetId, + driverManifestInput, type CreateDesktopStorybookDriverManifestOptions, type DesktopStorybookAgentManifest, type DesktopStorybookDriverManifest, @@ -46,9 +54,11 @@ import { import { NodeDesktopCommandRunner, STORYBOOK_VERBOSE } from './commandRunner.js'; import type { DesktopCommandRunner, PreparedDesktopCommand, RunningDesktopCommand } from './commandRunner.js'; import { formatDesktopStorybookSmokeTestSummary, runDesktopStorybookSmokeTests } from './smokeTests.js'; -import { matchesWdioTest, runWdioStoryTests, selectWdioStories } from '../testing/runWdioTests.js'; +import { runWdioStoryTests } from '../testing/runWdioTests.js'; +import { describeWdioCounts } from '../testing/laneVerdict.js'; +import { decideWdioSelection, describeWdioExclusions, listWdioSelection } from '../testing/selection.js'; +import type { WdioSelectionContext } from '../testing/selection.js'; import { resolveWdioOptions, type DesktopStorybookWdioOptions } from '../config/wdio.js'; -import { matchesDesktopPlatforms } from '@fluentui-react-native/desktop-driver/authoring'; type ResolvedDesktopStorybookInstance = DesktopStorybookInstance & { driverManifestPath?: string; @@ -352,45 +362,20 @@ export class DesktopStorybookCli { ? resolveWdioOptions({ ...initialSettings, input: connection.input }) : initialSettings; const manifest = await this.createStoryManifest(this.config, platform); - const selected = selectWdioStories(manifest, settings, platform); - if (selected.length === 0) { - throw new Error('No executable wdio story tests matched the requested selection.'); + const scope = this.scopeContext(platform); + const selection = decideWdioSelection(manifest, settings, { platform, scope }); + if (selection.stories.length === 0) { + throw new Error(`No executable wdio story tests matched the requested selection${describeWdioExclusions(selection)}.`); } if (options.list) { - const value = selected.map(({ id, sourcePath, wdio }) => ({ - id, - sourcePath, - ...(wdio?.cases - ? { - tests: wdio.cases - .filter( - ({ name, options: descriptor }) => - matchesWdioTest(name, settings.test) && - (!settings.input || descriptor.input === 'both' || descriptor.input === settings.input) && - matchesDesktopPlatforms(descriptor.platforms, platform), - ) - .map(({ name, options: descriptor }) => ({ - name, - input: descriptor.input, - inputSource: descriptor.inputSource, - requires: descriptor.requires ?? [], - platforms: descriptor.platforms, - })), - } - : wdio?.testNames - ? { tests: wdio.testNames.filter((name) => matchesWdioTest(name, settings.test)) } - : {}), - })); + const value = listWdioSelection(selection, settings); if (this.reporterMode === 'json' || this.reporterMode === 'ndjson') this.writeStructured(value); else this.output.write(`${JSON.stringify(value, null, 2)}\n`); return; } const activeConnection = connection ?? this.testConnection(platform, options); if (platform === 'macos' && !options.url) { - await this.writeMacOSApplicationLease( - path.join(this.config.projectRoot, 'storybook-desktop.generated', 'driver-manifest.macos.json'), - settings.timeoutMs, - ); + await this.writeMacOSApplicationLease(desktopStorybookDriverManifestPath(this.config.projectRoot, 'macos'), settings.timeoutMs); } const results = await runWdioStoryTests( { @@ -399,6 +384,7 @@ export class DesktopStorybookCli { config: this.config, manifest, platform, + scope, errorOutput: this.errorOutput, report: this.reporter, signal: this.services.controller.signal, @@ -406,10 +392,14 @@ export class DesktopStorybookCli { }, this.runner, ); - this.output.write( - `Ran ${results.length} executable wdio tests (${results.filter(({ status }) => status === 'passed').length} passed, ` + - `${results.filter(({ status }) => status === 'skipped').length} skipped).\n`, - ); + this.output.write(`Ran ${results.length} executable wdio tests (${describeWdioCounts(results)}).\n`); + } + + /** The scope domains and the values this endpoint's target holds, which declared case `scope` selection reads. */ + private scopeContext(platform: Platforms): NonNullable { + const domains = this.config.desktop?.testScopes ?? resolveDesktopScopeDomains(); + // This run's renderer wins over the host configuration, since `--paper` switches it. + return { domains, values: resolveDesktopTestScope(domains, this.config.getDesktopTarget(platform), this.renderer(platform)) }; } private testConnection( @@ -422,7 +412,7 @@ export class DesktopStorybookCli { } return { targetId: options.target, url: options.url }; } - const manifestPath = path.join(this.config.projectRoot, 'storybook-desktop.generated', `driver-manifest.${platform}.json`); + const manifestPath = desktopStorybookDriverManifestPath(this.config.projectRoot, platform); if (!fs.existsSync(manifestPath)) { throw new Error(`Start storybook driver --${platform} and launch the app before running storybook test.`); } @@ -452,16 +442,7 @@ export class DesktopStorybookCli { `The Storybook driver instance at ${manifestPath} serves an agent; restart storybook driver --${platform} without --agent to run storybook test.`, ); } - const input = - typeof manifest === 'object' && - manifest !== null && - 'wdio' in manifest && - typeof manifest.wdio === 'object' && - manifest.wdio !== null && - 'input' in manifest.wdio && - (manifest.wdio.input === 'accessible' || manifest.wdio.input === 'physical') - ? manifest.wdio.input - : undefined; + const input = driverManifestInput(manifest); return { targetId: manifest.targetId, url: loopbackUrl(manifest.driverPort), ...(input ? { input } : {}) }; } @@ -623,6 +604,7 @@ export class DesktopStorybookCli { driverUrl: loopbackUrl(instance.driverPort), manifest: await this.createStoryManifest(this.config, platform), platform, + scope: this.scopeContext(platform), targetId: this.targetId(platform), report: this.reporter, signal: this.services.controller.signal, @@ -722,7 +704,7 @@ export class DesktopStorybookCli { wdio?: CreateDesktopStorybookDriverManifestOptions['wdio'], ): Promise<{ manifest: DesktopStorybookDriverManifest; path: string }> { const storyManifest = await this.createStoryManifest(this.config, platform); - const outputPath = path.join(this.config.projectRoot, 'storybook-desktop.generated', `driver-manifest.${platform}.json`); + const outputPath = desktopStorybookDriverManifestPath(this.config.projectRoot, platform); const renderer = this.renderer(platform); const driverManifest = createDesktopStorybookDriverManifest({ ...(agent ? { agent } : {}), @@ -732,6 +714,7 @@ export class DesktopStorybookCli { nativeDriver, platform, renderer, + scope: this.scopeContext(platform), storyManifest, ...(wdio ? { wdio } : {}), }); diff --git a/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts b/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts index f7552ff94f..404f6a88d7 100644 --- a/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts +++ b/packages/agentic/storybook-desktop/src/cli/createDesktopStorybookCommand.ts @@ -26,6 +26,7 @@ import type { DesktopStorybookTestOptions, } from './DesktopStorybookCli.js'; import { DesktopStorybookCli } from './DesktopStorybookCli.js'; +import { resolveCommandInputLane, readDriverInstanceInput } from './inputLane.js'; import { loadDesktopStorybookConfig } from './loadConfig.js'; type PlatformFlags = { @@ -276,10 +277,27 @@ export async function runDesktopStorybookCli(argv: readonly string[] = process.a configured: options.config.desktop?.reporting?.events, }); options.reporterMode = reporterMode; + const actionOptions = action.opts(); + const platform = selectedPlatform(actionOptions) ?? options.config.platform; + const { projectRoot } = options.config; + const lane = + action.name() === 'test' || action.name() === 'smoke' + ? resolveCommandInputLane({ + command: action.name() as 'test' | 'smoke', + runsTests: action.name() === 'test' || actionOptions.mode === 'stories-and-tests', + flag: actionOptions.input, + environment: process.env.DESKTOP_INPUT, + driverInput: + action.name() === 'test' && platform && !actionOptions.url && !actionOptions.target + ? () => readDriverInstanceInput(projectRoot, platform) + : undefined, + }) + : undefined; session = await createDesktopReportSession({ ...options.config.desktop?.reporting, artifactsRoot: options.config.desktop?.reporting.artifactsRoot ?? path.join(options.config.projectRoot, 'artifacts', 'desktop'), kind: ['test', 'smoke'].includes(action.name()) ? 'tests' : 'lifecycle', + ...(lane ? { lane } : {}), reporterMode, eventLevel, // A long-lived command's readiness record is its only output; before readiness, a failure reports its summary. diff --git a/packages/agentic/storybook-desktop/src/cli/inputLane.test.ts b/packages/agentic/storybook-desktop/src/cli/inputLane.test.ts new file mode 100644 index 0000000000..1d9202f145 --- /dev/null +++ b/packages/agentic/storybook-desktop/src/cli/inputLane.test.ts @@ -0,0 +1,101 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { readDriverInstanceInput, resolveCommandInputLane } from './inputLane.js'; + +describe('command input lane', () => { + const driverInput = jest.fn((): 'accessible' | 'physical' | undefined => 'physical'); + beforeEach(() => driverInput.mockClear()); + + test('has no lane in compatibility mode', () => { + expect(resolveCommandInputLane({ command: 'test', runsTests: true, driverInput: () => undefined })).toBeUndefined(); + expect(resolveCommandInputLane({ command: 'test', runsTests: true })).toBeUndefined(); + }); + + test('a run that executes no authored tests has no lane to account for', () => { + expect(resolveCommandInputLane({ command: 'smoke', runsTests: false, flag: 'accessible', driverInput })).toBeUndefined(); + expect(driverInput).not.toHaveBeenCalled(); + }); + + test('prefers the flag, then the environment, then the driver instance', () => { + const request = { command: 'test', runsTests: true, driverInput } as const; + expect(resolveCommandInputLane({ ...request, flag: 'accessible', environment: 'physical' })).toMatchObject({ + input: 'accessible', + source: 'option', + }); + expect(resolveCommandInputLane({ ...request, environment: 'accessible' })).toMatchObject({ + input: 'accessible', + source: 'environment', + }); + expect(resolveCommandInputLane(request)).toMatchObject({ input: 'physical', source: 'driver' }); + }); + + test('does not read the driver instance when a flag or environment names the lane', () => { + resolveCommandInputLane({ command: 'test', runsTests: true, flag: 'physical', driverInput }); + resolveCommandInputLane({ command: 'test', runsTests: true, environment: 'physical', driverInput }); + expect(driverInput).not.toHaveBeenCalled(); + }); + + test('labels how an accessible lane is masked by who owns the driver, and never a physical lane', () => { + expect(resolveCommandInputLane({ command: 'smoke', runsTests: true, flag: 'accessible' })).toEqual({ + input: 'accessible', + source: 'option', + masking: 'host-masked', + policyEnforced: false, + }); + expect(resolveCommandInputLane({ command: 'test', runsTests: true, flag: 'accessible' })).toEqual({ + input: 'accessible', + source: 'option', + masking: 'verified-masked', + policyEnforced: false, + }); + expect(resolveCommandInputLane({ command: 'smoke', runsTests: true, flag: 'physical' })).toEqual({ + input: 'physical', + source: 'option', + policyEnforced: false, + }); + }); + + test.each([ + ['flag', { flag: 'both' }, '--input must be accessible or physical.'], + ['environment', { environment: 'Physical' }, 'DESKTOP_INPUT must be accessible or physical.'], + ['empty flag', { flag: '' }, '--input must be accessible or physical.'], + ['empty environment', { environment: '' }, 'DESKTOP_INPUT must be accessible or physical.'], + ] as const)('rejects an invalid %s', (_name, request, message) => { + expect(() => resolveCommandInputLane({ command: 'test', runsTests: true, ...request })).toThrow( + expect.objectContaining({ message, code: 'config' }), + ); + }); +}); + +describe('driver instance input', () => { + let root: string; + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'storybook-input-lane-')); + fs.mkdirSync(path.join(root, 'storybook-desktop.generated')); + }); + afterEach(() => fs.rmSync(root, { recursive: true, force: true })); + const write = (contents: string) => + fs.writeFileSync(path.join(root, 'storybook-desktop.generated', 'driver-manifest.win32.json'), contents); + + test('reads the lane a driver instance was started for', () => { + write(JSON.stringify({ wdio: { input: 'accessible' } })); + expect(readDriverInstanceInput(root, 'win32')).toBe('accessible'); + }); + + test.each([ + ['no lane', JSON.stringify({ wdio: {} })], + ['an unknown lane', JSON.stringify({ wdio: { input: 'both' } })], + ['no wdio settings', JSON.stringify({})], + ['malformed JSON', '{'], + ['a non-object manifest', 'null'], + ])('declares none for %s', (_name, contents) => { + write(contents); + expect(readDriverInstanceInput(root, 'win32')).toBeUndefined(); + }); + + test('declares none without a manifest for the endpoint', () => { + expect(readDriverInstanceInput(root, 'macos')).toBeUndefined(); + }); +}); diff --git a/packages/agentic/storybook-desktop/src/cli/inputLane.ts b/packages/agentic/storybook-desktop/src/cli/inputLane.ts new file mode 100644 index 0000000000..8a6e36dd39 --- /dev/null +++ b/packages/agentic/storybook-desktop/src/cli/inputLane.ts @@ -0,0 +1,42 @@ +import fs from 'node:fs'; + +import type { DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; +import { createDesktopInputLane, requestedDesktopInputLane } from '@fluentui-react-native/desktop-test/reporting'; +import type { DesktopInputLaneReport } from '@fluentui-react-native/desktop-test/reporting'; + +import type { Platforms } from '../config/platforms.js'; +import { desktopStorybookDriverManifestPath, driverManifestInput } from '../driver/driverManifest.js'; + +export type CommandInputLaneRequest = { + command: 'test' | 'smoke'; + /** `smoke` runs authored tests only in its stories-and-tests mode; a run without tests has no lane to account for. */ + runsTests: boolean; + flag?: string; + environment?: string; + /** The lane the attached driver instance was started for. Only `test` attaches to one, and only without `--url`. */ + driverInput?: () => DesktopInputLane | undefined; +}; + +/** + * The lane a `test` or `smoke` run holds its tests to, and where it came from: `--input`, then `DESKTOP_INPUT`, then + * the driver instance `test` attaches to. No lane is compatibility mode, which keeps the behavior from before lanes. + * An accessible lane's masking depends on who owns the driver: `smoke` starts it masked, while `test` attaches to one + * and verifies the mask before it runs anything. + */ +export function resolveCommandInputLane(request: CommandInputLaneRequest): DesktopInputLaneReport | undefined { + if (!request.runsTests) return undefined; + const masking = request.command === 'smoke' ? 'host-masked' : 'verified-masked'; + const requested = requestedDesktopInputLane({ flag: request.flag, environment: request.environment }); + if (requested) return createDesktopInputLane(requested.input, requested.source, masking); + const input = request.driverInput?.(); + return input ? createDesktopInputLane(input, 'driver', masking) : undefined; +} + +/** The lane the running driver instance for an endpoint declares. A missing or unreadable manifest declares none. */ +export function readDriverInstanceInput(projectRoot: string, platform: Platforms): DesktopInputLane | undefined { + try { + return driverManifestInput(JSON.parse(fs.readFileSync(desktopStorybookDriverManifestPath(projectRoot, platform), 'utf8'))); + } catch { + return undefined; + } +} diff --git a/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts b/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts index da8b2d0bac..c07b9ee7b2 100644 --- a/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts +++ b/packages/agentic/storybook-desktop/src/cli/smokeTests.test.ts @@ -4,10 +4,13 @@ import path from 'node:path'; import type { DesktopStoryManifest, DesktopStoryManifestEntry } from '@fluentui-react-native/desktop-driver'; import type { DesktopStoryRunResult } from '@fluentui-react-native/desktop-driver/authoring'; +import { validateDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopReportWriter } from '@fluentui-react-native/desktop-test/reporting'; import { makeDesktopStorybookConfig } from '../config/makeDesktopStorybookConfig.js'; import type { runWdioStoryTests } from '../testing/runWdioTests.js'; import { WdioStoryTestRunError } from '../testing/runWdioTests.js'; +import { csfTestId } from '../testing/selection.js'; import { formatDesktopStorybookSmokeTestSummary, runDesktopStorybookSmokeTests, @@ -31,6 +34,32 @@ function story(id: string, wdio = false): DesktopStoryManifestEntry { }; } +function declaredStory(id: string, cases: readonly (readonly [name: string, options: unknown])[]): DesktopStoryManifestEntry { + return { + ...story(id), + tests: undefined, + wdio: { + exportName: 'Default', + digest: 'callback', + testNames: cases.map(([name]) => name), + cases: cases.map(([name, descriptor]) => ({ name, options: validateDesktopCaseOptions(descriptor) })), + }, + }; +} + +function fakeReport(root: string) { + const emit = jest.fn(); + const report = { + runId: 'run', + root, + emit, + artifactPath: (name: string) => path.join(root, name), + failure: jest.fn(), + fail: jest.fn(), + } as unknown as DesktopReportWriter; + return { report, emit }; +} + const manifest: DesktopStoryManifest = { endpoint: 'windows', entries: [story(buttonId)], @@ -290,4 +319,99 @@ describe('runDesktopStorybookSmokeTests', () => { await expect(runDesktopStorybookSmokeTests(options, connect)).rejects.toThrow('No desktop story tests'); expect(connect).not.toHaveBeenCalled(); }); + + describe('input lanes', () => { + test('announces every selected case up front and runs each story against that announcement', async () => { + const { report, emit } = fakeReport(root); + options.report = report; + options.input = 'accessible'; + options.scope = { domains: { 'rn.renderer': ['fabric', 'paper'] }, values: { 'rn.renderer': 'fabric' } }; + options.manifest = { + ...manifest, + entries: [ + declaredStory(buttonId, [ + ['plain', {}], + ['typed', { requires: ['keyboard'] }], + ]), + declaredStory(inputId, [['typing', { requires: ['accessibility-click'] }]]), + ], + }; + const runWdio: jest.MockedFunction = jest.fn(async ({ story }) => [ + { storyId: story!, testName: 'ran', status: 'passed' as const, durationMs: 1 }, + ]); + + await runDesktopStorybookSmokeTests(options, jest.fn(), runWdio); + + const events = emit.mock.calls.map(([event]) => event); + expect(events.filter(({ kind }) => kind === 'selection')).toEqual([ + { kind: 'selection', payload: { excluded: { platform: 0, scope: 0, input: 1 } } }, + ]); + expect(events.filter(({ kind }) => kind === 'work-selected').map(({ payload }) => payload.workId)).toEqual([ + `smoke:${buttonId}`, + `smoke:${inputId}`, + csfTestId(buttonId, 'plain'), + csfTestId(inputId, 'typing'), + ]); + expect(runWdio).toHaveBeenCalledWith( + expect.objectContaining({ story: buttonId, input: 'accessible', scope: options.scope, announced: true, optional: true }), + undefined, + ); + }); + + test('rejects legacy callbacks before connecting', async () => { + options.input = 'physical'; + options.manifest = { ...manifest, entries: [{ ...story(buttonId, true), tests: undefined }] }; + const connect = jest.fn(); + await expect(runDesktopStorybookSmokeTests(options, connect)).rejects.toThrow( + `Input lane "physical" cannot run legacy wdio callbacks; declare them with desktopIt: ${buttonId}.`, + ); + expect(connect).not.toHaveBeenCalled(); + }); + + test('states a lane selection that left every case out and explains why', async () => { + const { report, emit } = fakeReport(root); + options.report = report; + options.input = 'accessible'; + options.manifest = { ...manifest, entries: [declaredStory(buttonId, [['typed', { requires: ['keyboard'] }]])] }; + await expect(runDesktopStorybookSmokeTests(options, jest.fn())).rejects.toThrow( + 'No desktop story tests matched the smoke selection (excluded 0 by platform, 0 by scope, 1 by input).', + ); + expect(emit).toHaveBeenCalledWith({ kind: 'selection', payload: { excluded: { platform: 0, scope: 0, input: 1 } } }); + }); + + test('a lane that executed no wdio test is not a passing run even when static plans passed', async () => { + options.input = 'accessible'; + options.manifest = { ...manifest, entries: [{ ...story(buttonId), wdio: declaredStory(buttonId, [['plain', {}]]).wdio }] }; + const connect: DesktopStorybookSmokeConnector = async () => ({ + delete: async () => undefined, + listStories: async () => options.manifest, + runStoryTests: async () => planResult(), + }); + const runWdio = jest.fn(async () => [ + { storyId: buttonId, testName: 'plain', status: 'unsupported' as const, gap: 'unsupported' as const, durationMs: 0 }, + ]); + await expect(runDesktopStorybookSmokeTests(options, connect, runWdio)).rejects.toThrow( + 'Input lane "accessible" executed no wdio story tests; it is not a passing run.', + ); + }); + }); + + test('does not pass a cohort that only skipped or was unsupported', async () => { + options.manifest = { ...manifest, entries: [{ ...story(buttonId, true), tests: undefined }] }; + const runWdio = jest.fn(async () => [ + { storyId: buttonId, status: 'unsupported' as const, gap: 'unsupported' as const, durationMs: 0 }, + ]); + await expect(runDesktopStorybookSmokeTests(options, jest.fn(), runWdio)).rejects.toThrow('only skipped or unsupported'); + }); + + test('breaks unsupported down in the summary only when something is unsupported', () => { + const wdio = [{ storyId: buttonId, status: 'passed' as const, durationMs: 1 }]; + expect(formatDesktopStorybookSmokeTestSummary({ ...planResult(), wdio })).toBe('Ran 2 desktop story tests (2 passed, 0 skipped).'); + expect( + formatDesktopStorybookSmokeTestSummary({ + ...planResult(), + wdio: [...wdio, { storyId: inputId, status: 'unsupported' as const, gap: 'unsupported' as const, durationMs: 0 }], + }), + ).toBe('Ran 3 desktop story tests (2 passed, 0 skipped, 1 unsupported (1 declared, 0 quarantined)).'); + }); }); diff --git a/packages/agentic/storybook-desktop/src/cli/smokeTests.ts b/packages/agentic/storybook-desktop/src/cli/smokeTests.ts index afa53f720f..c45a586c70 100644 --- a/packages/agentic/storybook-desktop/src/cli/smokeTests.ts +++ b/packages/agentic/storybook-desktop/src/cli/smokeTests.ts @@ -12,7 +12,9 @@ import type { DesktopWebdriverOptions, DesktopWebdriverSession } from '@fluentui import type { DesktopStorybookConfig } from '../config/makeDesktopStorybookConfig.js'; import type { Platforms } from '../config/platforms.js'; import { resolveWdioOptions } from '../config/wdio.js'; -import { runWdioStoryTests, selectWdioStories, WdioStoryTestRunError, type WdioStoryTestResult } from '../testing/runWdioTests.js'; +import { describeWdioCounts, hasExecutedWdioCase } from '../testing/laneVerdict.js'; +import { runWdioStoryTests, WdioStoryTestRunError, type WdioStoryTestResult } from '../testing/runWdioTests.js'; +import { decideWdioSelection, describeWdioExclusions, plannedWdioWork, type WdioSelectionContext } from '../testing/selection.js'; import { writeDesktopStorybookFailure, type DesktopStorybookErrorOutput } from '../../config/diagnostics.cjs'; import type { DesktopCommandRunner } from './commandRunner.js'; @@ -22,6 +24,8 @@ export type DesktopStorybookSmokeConnector = (options: DesktopWebdriverOptions) export type DesktopStorybookSmokeTestOptions = { input?: 'accessible' | 'physical'; + /** The scope domains and values of this target, for case `scope` selection. */ + scope?: WdioSelectionContext['scope']; artifactsRoot?: string; config: DesktopStorybookConfig; driverUrl: string; @@ -60,10 +64,16 @@ export async function runDesktopStorybookSmokeTests( const planStoryIds = new Set( selectDesktopStoryTests(manifest, platform, { story: storyPattern, tag: testTag ?? 'desktop-e2e' }).map(({ entry }) => entry.id), ); - const wdioStoryIds = new Set(selectWdioStories(manifest, wdioSettings, platform).map(({ id }) => id)); + const wdioSelection = decideWdioSelection(manifest, wdioSettings, { platform, scope: options.scope }); + const wdioStoryIds = new Set(wdioSelection.stories.map(({ id }) => id)); + const { platform: byPlatform, scope: byScope, input: byInput } = wdioSelection.excluded; + // A lane always states what it excluded, even when nothing; compatibility mode only when something was excluded. + if (options.report && (wdioSettings.input || byPlatform + byScope + byInput > 0)) { + options.report.emit({ kind: 'selection', payload: { excluded: wdioSelection.excluded } }); + } const storyIds = [...new Set([...planStoryIds, ...wdioStoryIds])].sort(); if (storyIds.length === 0) { - throw new Error('No desktop story tests matched the smoke selection.'); + throw new Error(`No desktop story tests matched the smoke selection${describeWdioExclusions(wdioSelection)}.`); } const artifacts = new ArtifactManager( options.artifactsRoot ?? @@ -77,6 +87,8 @@ export async function runDesktopStorybookSmokeTests( const failures: unknown[] = []; const staticFailures = new Map(); for (const storyId of storyIds) options.report?.emit({ kind: 'work-selected', payload: { workId: `smoke:${storyId}`, label: storyId } }); + // Every selected case is announced up front, so a fail-fast stop leaves the remaining cases accounted for as not run. + for (const payload of plannedWdioWork(wdioSelection, wdioSettings)) options.report?.emit({ kind: 'work-selected', payload }); for (const storyId of storyIds) { const scope = `smoke:${storyId}`; @@ -106,6 +118,7 @@ export async function runDesktopStorybookSmokeTests( config, manifest, platform, + scope: options.scope, story: storyId, targetId: options.targetId, url: options.driverUrl, @@ -113,6 +126,7 @@ export async function runDesktopStorybookSmokeTests( report: options.report, signal: options.signal, optional: true, + announced: true, }, options.commandRunner, )), @@ -154,9 +168,18 @@ export async function runDesktopStorybookSmokeTests( } failures.push(new Error('Executable smoke tests reported failures.')); } - const allSkipped = tests.length + wdio.length > 0 && [...tests, ...wdio].every(({ status }) => status === 'skipped'); + const allSkipped = + tests.length + wdio.length > 0 && [...tests, ...wdio].every(({ status }) => status === 'skipped' || status === 'unsupported'); if (allSkipped && failures.length === 0) - failures.push(new Error('Required desktop smoke cohort only skipped tests; it is not a passing run.')); + failures.push(new Error('Required desktop smoke cohort only skipped or unsupported tests; it is not a passing run.')); + // Smoke runs one story at a time as optional, so the lane's requirement to execute something is judged for the cohort here. + if (wdioSettings.input && failures.length === 0 && !hasExecutedWdioCase(wdio)) { + failures.push( + new Error( + `Input lane "${wdioSettings.input}" executed no wdio story tests${describeWdioExclusions(wdioSelection)}; it is not a passing run.`, + ), + ); + } const result: DesktopStorybookSmokeTestResult = { endpoint: platform, finishedAt: new Date().toISOString(), @@ -289,9 +312,7 @@ async function runStoryPlans( export function formatDesktopStorybookSmokeTestSummary(result: DesktopStorybookSmokeTestResult): string { const results = [...result.tests, ...(result.wdio ?? [])]; - const passed = results.filter(({ status }) => status === 'passed').length; - const skipped = results.filter(({ status }) => status === 'skipped').length; - return `Ran ${results.length} desktop story tests (${passed} passed, ${skipped} skipped).`; + return `Ran ${results.length} desktop story tests (${describeWdioCounts(results)}).`; } function errorMessage(error: unknown): string { diff --git a/packages/agentic/storybook-desktop/src/config/integration.ts b/packages/agentic/storybook-desktop/src/config/integration.ts index 3d1b8f00c3..6bf1071cde 100644 --- a/packages/agentic/storybook-desktop/src/config/integration.ts +++ b/packages/agentic/storybook-desktop/src/config/integration.ts @@ -32,7 +32,7 @@ export function storybook(options: DesktopStorybookConfigOptions = {}): Storyboo configure(projectRoot, desktop) { return new DesktopStorybookConfig({ ...options, projectRoot }, desktop); }, - async run({ config, target, supervisor, action, agent, reporterMode, onReady }) { + async run({ config, target, supervisor, action, agent, input, reporterMode, onReady }) { const { DesktopStorybookCli } = await import('@fluentui-react-native/storybook-desktop/cli'); const api = new DesktopStorybookCli(this.configure(config.projectRoot, config), { paper: target.endpoint === 'macos' && target.host.renderer === 'paper', @@ -43,7 +43,7 @@ export function storybook(options: DesktopStorybookConfigOptions = {}): Storyboo ...(onReady ? { onReady } : {}), }); if (action === 'serve') await api.driver(target.endpoint, agent ? { agent } : {}); - else if (action === 'test') await api.smoke(target.endpoint, { mode: 'stories-and-tests' }); + else if (action === 'test') await api.smoke(target.endpoint, { mode: 'stories-and-tests', ...(input ? { input } : {}) }); else await api[action](target.endpoint); }, }; diff --git a/packages/agentic/storybook-desktop/src/config/makeDesktopStorybookConfig.ts b/packages/agentic/storybook-desktop/src/config/makeDesktopStorybookConfig.ts index f65a554091..b61e6f19e6 100644 --- a/packages/agentic/storybook-desktop/src/config/makeDesktopStorybookConfig.ts +++ b/packages/agentic/storybook-desktop/src/config/makeDesktopStorybookConfig.ts @@ -116,8 +116,11 @@ export type DesktopStorybookConfigOptions = StorySettings & { */ platformOptions?: DesktopPlatformOptionsMap; - /** Defaults for executable story callbacks, run by `storybook-desktop test`. */ - wdio?: DesktopStorybookWdioOptions; + /** + * Defaults for executable story callbacks, run by `storybook-desktop test`. The input lane is chosen per run with + * `--input` or `DESKTOP_INPUT`, so it cannot be configured here. + */ + wdio?: Omit; }; export type ResolvedPackage = { @@ -532,6 +535,10 @@ function defaultSmokeOptions(config: DesktopStorybookConfig, platform: Platforms } function normalizeConfig(config: DesktopStorybookConfigOptions, projectRoot: string): DesktopStorybookConfigOptions { + // A configured lane would run tests the report session never holds to a lane, so the lane has one per-run source. + if (config.wdio && 'input' in config.wdio) { + throw new TypeError('wdio.input is chosen per run with --input or DESKTOP_INPUT; it cannot be configured.'); + } const wdio = resolveWdioOptions(config.wdio); validatePlatforms(config.platforms, 'config'); validatePlatformSettings(config.platformSettings, 'config'); diff --git a/packages/agentic/storybook-desktop/src/config/wdio.test.ts b/packages/agentic/storybook-desktop/src/config/wdio.test.ts index 996fdf90c4..e275ae5b52 100644 --- a/packages/agentic/storybook-desktop/src/config/wdio.test.ts +++ b/packages/agentic/storybook-desktop/src/config/wdio.test.ts @@ -25,6 +25,11 @@ describe('Storybook-owned wdio configuration', () => { expect(() => resolveWdioOptions({ tag: ' ' })).toThrow('wdio.tag'); }); + test('rejects a configured input lane, which is chosen per run', () => { + // @ts-expect-error `input` is not a configurable default. + expect(() => makeDesktopStorybookConfig({ wdio: { input: 'accessible' } })).toThrow('wdio.input is chosen per run'); + }); + test('resolves an explicit input lane and rejects conflicting click configuration', () => { const previous = process.env.DESKTOP_INPUT; delete process.env.DESKTOP_INPUT; diff --git a/packages/agentic/storybook-desktop/src/driver/driverManifest.ts b/packages/agentic/storybook-desktop/src/driver/driverManifest.ts index 69dd15764d..3e4ac5b7a0 100644 --- a/packages/agentic/storybook-desktop/src/driver/driverManifest.ts +++ b/packages/agentic/storybook-desktop/src/driver/driverManifest.ts @@ -3,12 +3,14 @@ import fs from 'node:fs'; import path from 'node:path'; import type { DesktopStoryManifest, NativeDesktopApplicationDescriptor, NativeDriverArtifact } from '@fluentui-react-native/desktop-driver'; +import type { DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; import { selectDesktopRenderer } from '@fluentui-react-native/desktop-test/config'; import type { DesktopStorybookConfig } from '../config/makeDesktopStorybookConfig.js'; import type { DesktopStorybookInstance } from '../config/instance.js'; import type { Platforms } from '../config/platforms.js'; import type { DesktopStorybookWdioOptions } from '../config/wdio.js'; +import type { WdioSelectionContext } from '../testing/selection.js'; /** Asks the channel service to protect its driver with a grant authority and publish one agent's connection. */ export type DesktopStorybookAgentManifest = { @@ -43,6 +45,11 @@ export type DesktopStorybookDriverManifest = { portablePlanDigest: string; renderer: 'fabric' | 'paper'; schemaVersion: 2; + /** + * The scope domains and the values this target holds, which declared test `scope` selection reads. The driver's + * CLI resolves them from the full desktop configuration, which a smoke control process does not have. + */ + scope?: NonNullable; storyManifest: DesktopStoryManifest; storybookPort: number; targetId: string; @@ -58,6 +65,7 @@ export type CreateDesktopStorybookDriverManifestOptions = { nativeDriver: NativeDriverArtifact; platform: Platforms; renderer?: DesktopStorybookDriverManifest['renderer']; + scope?: DesktopStorybookDriverManifest['scope']; storyManifest: DesktopStoryManifest; wdio?: DesktopStorybookWdioOptions; }; @@ -81,6 +89,7 @@ export function createDesktopStorybookDriverManifest({ nativeDriver, platform, renderer = platform === 'win32' ? 'paper' : 'fabric', + scope, storyManifest, wdio, }: CreateDesktopStorybookDriverManifestOptions): DesktopStorybookDriverManifest { @@ -119,6 +128,9 @@ export function createDesktopStorybookDriverManifest({ portablePlanDigest: storyManifest.portablePlanDigest, renderer, schemaVersion: 2, + ...(scope + ? { scope: Object.freeze({ domains: Object.freeze({ ...scope.domains }), values: Object.freeze({ ...scope.values }) }) } + : {}), storyManifest, storybookPort: instance.storybookPort, targetId: desktopStorybookTargetId(config, platform, renderer), @@ -136,3 +148,15 @@ export function writeDesktopStorybookDriverManifest(manifest: DesktopStorybookDr fs.writeFileSync(outputPath, content); } } + +/** Where the running driver instance for an endpoint publishes its manifest. */ +export function desktopStorybookDriverManifestPath(projectRoot: string, platform: Platforms): string { + return path.join(projectRoot, 'storybook-desktop.generated', `driver-manifest.${platform}.json`); +} + +/** The input lane a driver instance was started for, read from any parsed manifest; absent when it declares none. */ +export function driverManifestInput(manifest: unknown): DesktopInputLane | undefined { + const wdio = typeof manifest === 'object' && manifest !== null && 'wdio' in manifest ? manifest.wdio : undefined; + const input = typeof wdio === 'object' && wdio !== null && 'input' in wdio ? wdio.input : undefined; + return input === 'accessible' || input === 'physical' ? input : undefined; +} diff --git a/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts b/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts index bdea026b70..c7bf96822d 100644 --- a/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts +++ b/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts @@ -94,6 +94,64 @@ describe('desktop Storybook server integration', () => { expect(output.join('')).toContain('WebDriver '); }); + test.each([ + ['accessible', { keyboard: false, physicalClick: false, wheel: false }], + ['physical', { keyboard: true, physicalClick: true, wheel: true }], + ] as const)('starts a driver for the %s input lane with matching physical input features', async (input, expected) => { + const [storybookPort, driverPort] = await Promise.all([getAvailablePort(), getAvailablePort()]); + const temporaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-storybook-driver-')); + const manifestPath = path.join(temporaryDirectory, 'driver-manifest.json'); + const projectRoot = path.resolve(__dirname, '../../../../../apps/storybook'); + fs.writeFileSync( + manifestPath, + JSON.stringify({ ...createIntegrationManifest(temporaryDirectory, storybookPort, driverPort), wdio: { input } }), + ); + const sink = { write: () => true }; + const reporter = new DesktopRunReporter({ + artifactsRoot: temporaryDirectory, + kind: 'lifecycle', + output: sink, + errorOutput: sink, + summary: false, + }); + const transport = await startReportTransport(reporter); + const runner = new NodeDesktopCommandRunner({ reporter, environment: transport.environment }); + const child = runner.start({ + command: process.execPath, + args: [path.resolve(__dirname, '../../config/server-runner.cjs')], + cwd: projectRoot, + env: { + STORYBOOK_CONFIG_PATH: path.join(projectRoot, 'src'), + STORYBOOK_DRIVER_MANIFEST: manifestPath, + STORYBOOK_NATIVE_DRIVER_FAKE: '1', + STORYBOOK_PROJECT_ROOT: projectRoot, + STORYBOOK_SMOKE_MODE: 'stories-and-tests', + STORYBOOK_WS_PORT: String(storybookPort), + }, + }); + + try { + await waitForResponse(loopbackUrl(driverPort, '/status')); + const client = createDesktopDriverClient({ url: loopbackUrl(driverPort, '') }); + const session = await client.newSession({ + alwaysMatch: { platformName: 'windows', 'furn:target': 'integration-windows', 'furn:launchMode': 'attach' }, + }); + // The accessible lane trusts this report, so the driver itself must show physical input disabled. + expect(session.capabilities['furn:features']).toMatchObject(expected); + await session.delete(); + } finally { + try { + await child.stop(); + await child.completed; + await runner.close(); + await transport.close(); + await reporter.finish(); + } finally { + fs.rmSync(temporaryDirectory, { force: true, recursive: true }); + } + } + }); + test('refuses an agent connection from the fake driver', async () => { const [storybookPort, driverPort] = await Promise.all([getAvailablePort(), getAvailablePort()]); const temporaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-storybook-driver-')); diff --git a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts index 0a5ff18bb4..16766657bb 100644 --- a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts +++ b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.test.ts @@ -285,4 +285,115 @@ describe('extractWdioTests', () => { expect(digest(source.replace("'test'", "'changed'"))).not.toBe(digest(source)); expect(digest(source.replace('console.log', '/* source change */ console.log'))).not.toBe(digest(source)); }); + + describe('direct physical input', () => { + const desktopCase = (body: string, options?: string) => + `import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; + export const Example = { wdio: [desktopIt('case', ${options === undefined ? '' : `${options}, `}async (context) => { const { browser } = context; ${body} })] };`; + const key = `{ type: 'key', id: 'keyboard', actions: [{ type: 'keyDown', value: 'a' }] }`; + const pointer = `{ type: 'pointer', id: 'mouse', actions: [{ type: 'pointerMove', x: 1, y: 1 }] }`; + const wheel = `{ type: 'wheel', id: 'wheel', actions: [{ type: 'scroll', deltaY: 10 }] }`; + const pause = `{ type: 'none', id: 'pause', actions: [{ type: 'pause', duration: 100 }] }`; + + test.each([ + ['browser.keys', `await browser.keys('a');`, /browser\.keys sends physical keyboard input.*requires: \['keyboard'\]/], + ['an optional browser.keys', `await browser?.keys('a');`, /browser\.keys sends physical keyboard input/], + ['a computed browser.keys', `await browser['keys']('a');`, /browser\.keys sends physical keyboard input/], + ['a keys call through the context', `await context.browser.keys('a');`, /browser\.keys sends physical keyboard input/], + ['setValue', `await (await browser.$('~input')).setValue('a');`, /setValue sends physical keyboard input/], + ['addValue', `await (await browser.$('~input')).addValue('a');`, /addValue sends physical keyboard input/], + ['clearValue', `await (await browser.$('~input')).clearValue();`, /clearValue sends physical keyboard input/], + ['key actions', `await browser.performActions([${key}]);`, /performActions sends physical keyboard input/], + ['pointer actions', `await browser.performActions([${pointer}]);`, /sends physical pointer input.*requires: \['physical-click'\]/], + ['wheel actions', `await browser.performActions([${wheel}]);`, /sends physical wheel input.*requires: \['wheel'\]/], + [ + 'actions with several physical sources', + `await browser.performActions([${wheel}, ${pause}, ${key}]);`, + /sends physical keyboard and wheel input.*requires: \['keyboard', 'wheel'\]/, + ], + [ + 'actions held in a constant', + `const actions = [${pointer}]; await browser.performActions(actions);`, + /sends physical pointer input/, + ], + ['an action builder', `await browser.action('key').down('a').perform();`, /browser\.action sends physical keyboard input/], + [ + 'a call inside a nested function', + `await browser.waitUntil(async () => { await browser.keys('a'); return true; });`, + /browser\.keys sends physical keyboard input/, + ], + [ + 'actions built elsewhere', + `const buildActions = () => []; await browser.performActions(buildActions());`, + /performActions with an argument this check cannot read/, + ], + ['actions held in a variable that changes', `let actions = []; actions = [1]; await browser.performActions(actions);`, /cannot read/], + ['actions with a spread source', `const sources = []; await browser.performActions([...sources]);`, /cannot read/], + [ + 'actions with a source that has a spread', + `const source = {}; await browser.performActions([{ ...source, id: 'a' }]);`, + /cannot read/, + ], + ['actions without an argument', `await browser.performActions();`, /cannot read/], + [ + 'an action builder with a variable type', + `await browser.action(context.type).perform();`, + /browser\.action with an argument this check cannot read/, + ], + ])('rejects %s in a case that also runs in the accessible lane', (_name, body, message) => { + expect(() => extractWdioTests(desktopCase(body), sourceFile)).toThrow(message); + }); + + test.each([ + ['an explicit both', `{ input: 'both' }`], + ['an explicit accessible case', `{ input: 'accessible' }`], + ['an accessibility click', `{ requires: ['accessibility-click'] }`], + ['neutral requirements', `{ requires: ['click', 'focus'] }`], + ])('rejects the keyboard in %s with the case name and source location', (_name, options) => { + let error: Error | undefined; + try { + extractWdioTests(desktopCase(`await browser.keys('a');`, options), sourceFile); + } catch (caught) { + error = caught as Error; + } + expect(error?.message).toContain('"case" runs in the accessible input lane'); + expect(error?.message).toContain(sourceFile); + }); + + test('suggests the declaration that matches how the case chose its input', () => { + expect(() => extractWdioTests(desktopCase(`await browser.keys('a');`, `{ input: 'both' }`), sourceFile)).toThrow( + /Declare input: 'physical'/, + ); + expect(() => extractWdioTests(desktopCase(`await browser.keys('a');`), sourceFile)).toThrow(/Declare requires: \['keyboard'\]/); + }); + + test.each([ + ['derives physical from keyboard', `{ requires: ['keyboard'] }`], + ['derives physical from a physical click', `{ requires: ['physical-click', 'wheel'] }`], + ['declares physical', `{ input: 'physical' }`], + ])('allows direct input in a case that %s', (_name, options) => { + const body = `await browser.keys('a'); await (await browser.$('~input')).setValue('a'); await browser.performActions([${key}, ${pointer}, ${wheel}]);`; + expect(() => extractWdioTests(desktopCase(body, options), sourceFile)).not.toThrow(); + }); + + test('allows calls that do not send physical input', () => { + const body = ` + const names = Object.keys({ a: 1 }); + const map = new Map([[1, 2]]); + for (const entry of map.keys()) names.push(String(entry)); + await browser.performActions([${pause}]); + await browser.performActions([]); + const store = { setValueLater: () => {} }; + store.setValueLater(); + await (await browser.$('~button')).click(); + `; + expect(() => extractWdioTests(desktopCase(body), sourceFile)).not.toThrow(); + }); + + test('does not read legacy callbacks, which declare no input', () => { + expect(() => + extractWdioTests(`export const Example = { wdio: { legacy: async ({ browser }) => { await browser.keys('a'); } } };`, sourceFile), + ).not.toThrow(); + }); + }); }); diff --git a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts index 58e2eaf30f..4bf2066810 100644 --- a/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts +++ b/packages/agentic/storybook-desktop/src/testing/extractWdioTests.ts @@ -6,7 +6,7 @@ import type { ResolvedDesktopCaseOptions } from '@fluentui-react-native/desktop- import storyTests from '../../config/transform-story-tests.cjs'; -const { collectWdioTests, compileWdioTest } = storyTests; +const { collectWdioTests, compileWdioTest, rejectDirectInput } = storyTests; export type ExtractedWdioStory = { digest: string; @@ -26,11 +26,18 @@ export function extractWdioTests(code: string, sourceFile: string): ReadonlyMap< visitor: { Program(program: NodePath) { for (const [exportName, { callbacks, value }] of collectWdioTests(program)) { - const compiled = callbacks.map(({ callback, name, options }) => ({ - ...(name === undefined ? {} : { name }), - code: compileWdioTest(callback, sourceFile, code), - ...(options !== undefined ? { options: validateDesktopCaseOptions(options, `${sourceFile}.wdio`) } : {}), - })); + const compiled = callbacks.map(({ callback, name, options }) => { + const compiledCode = compileWdioTest(callback, sourceFile, code); + const descriptor = options === undefined ? undefined : validateDesktopCaseOptions(options, `${sourceFile}.wdio`); + if (descriptor !== undefined) { + rejectDirectInput(callback, { name, input: descriptor.input, inputSource: descriptor.inputSource }); + } + return { + ...(name === undefined ? {} : { name }), + code: compiledCode, + ...(descriptor === undefined ? {} : { options: descriptor }), + }; + }); const source = code.slice(value.node.start!, value.node.end!); tests.set(exportName, { tests: compiled, diff --git a/packages/agentic/storybook-desktop/src/testing/laneVerdict.test.ts b/packages/agentic/storybook-desktop/src/testing/laneVerdict.test.ts new file mode 100644 index 0000000000..845858dd7f --- /dev/null +++ b/packages/agentic/storybook-desktop/src/testing/laneVerdict.test.ts @@ -0,0 +1,75 @@ +import { assessWdioCohort, hasExecutedWdioCase, describeWdioCounts, type WdioCaseOutcome } from './laneVerdict.js'; + +const result = (testName: string, status: WdioCaseOutcome['status'], gap?: WdioCaseOutcome['gap']): WdioCaseOutcome => ({ + storyId: 'story', + testName, + status, + ...(gap ? { gap } : {}), +}); + +describe('WDIO cohort verdict', () => { + test('a compatibility cohort needs one executed test unless it is optional', () => { + expect(assessWdioCohort({ results: [result('a', 'passed'), result('b', 'skipped')], selected: 2 })).toMatchObject({ + executed: true, + incomplete: false, + }); + expect(assessWdioCohort({ results: [result('a', 'skipped')], selected: 1 })).toMatchObject({ + incomplete: true, + message: expect.stringContaining('no selected test executed'), + }); + expect(assessWdioCohort({ results: [result('a', 'unsupported')], selected: 1 })).toMatchObject({ incomplete: true }); + expect(assessWdioCohort({ results: [result('a', 'unsupported')], selected: 1, optional: true })).toMatchObject({ incomplete: false }); + expect(assessWdioCohort({ results: [result('a', 'passed'), result('b', 'unsupported')], selected: 2 })).toMatchObject({ + incomplete: false, + }); + }); + + test('a lane rejects a skip and an undeclared unsupported result, and names them', () => { + for (const status of ['skipped', 'unsupported'] as const) { + expect(assessWdioCohort({ results: [result('a', 'passed'), result('b', status)], selected: 2, lane: 'accessible' })).toMatchObject({ + executed: true, + incomplete: true, + message: expect.stringContaining('the accessible lane could not run 1 selected tests that have no declared gap (story / b)'), + }); + } + }); + + test('a lane accepts declared gaps and optional never masks an undeclared one', () => { + const declared = [result('a', 'passed'), result('b', 'unsupported', 'unsupported'), result('c', 'unsupported', 'quarantine')]; + expect(assessWdioCohort({ results: declared, selected: 3, lane: 'physical' })).toMatchObject({ incomplete: false }); + expect( + assessWdioCohort({ results: [result('a', 'unsupported', 'unsupported')], selected: 1, lane: 'physical', optional: true }), + ).toMatchObject({ executed: false, incomplete: false }); + expect(assessWdioCohort({ results: [result('a', 'unsupported')], selected: 1, lane: 'physical', optional: true })).toMatchObject({ + incomplete: true, + }); + expect(assessWdioCohort({ results: [result('a', 'unsupported', 'unsupported')], selected: 1, lane: 'physical' })).toMatchObject({ + incomplete: true, + message: expect.stringContaining('no selected test executed'), + }); + }); + + test('selected work that never finished is incomplete in every mode', () => { + expect(assessWdioCohort({ results: [result('a', 'passed')], selected: 3 })).toMatchObject({ + incomplete: true, + message: expect.stringContaining('2 of 3 selected tests did not finish'), + }); + }); + + test('counts only passes and failures as executed', () => { + expect(hasExecutedWdioCase([result('a', 'skipped'), result('b', 'unsupported')])).toBe(false); + expect(hasExecutedWdioCase([result('a', 'failed')])).toBe(true); + }); + + test('summarizes counts and breaks unsupported down only when present', () => { + expect(describeWdioCounts([result('a', 'passed'), result('b', 'skipped')])).toBe('1 passed, 1 skipped'); + expect( + describeWdioCounts([ + result('a', 'passed'), + result('b', 'unsupported', 'unsupported'), + result('c', 'unsupported', 'quarantine'), + result('d', 'unsupported'), + ]), + ).toBe('1 passed, 0 skipped, 3 unsupported (1 declared, 1 quarantined)'); + }); +}); diff --git a/packages/agentic/storybook-desktop/src/testing/laneVerdict.ts b/packages/agentic/storybook-desktop/src/testing/laneVerdict.ts new file mode 100644 index 0000000000..e33f24a8fc --- /dev/null +++ b/packages/agentic/storybook-desktop/src/testing/laneVerdict.ts @@ -0,0 +1,72 @@ +import type { DesktopInputLane } from '@fluentui-react-native/desktop-driver/authoring'; +import { describeDesktopUnsupported } from '@fluentui-react-native/desktop-test/reporting'; + +export type WdioCaseOutcome = { + storyId: string; + testName?: string; + status: 'passed' | 'failed' | 'skipped' | 'unsupported'; + gap?: 'unsupported' | 'quarantine'; +}; + +export type WdioCohortVerdict = { + /** At least one selected case ran to a pass or a failure. */ + executed: boolean; + incomplete: boolean; + /** Why the cohort is incomplete. */ + message?: string; +}; + +export function hasExecutedWdioCase(results: readonly Pick[]): boolean { + return results.some(({ status }) => status === 'passed' || status === 'failed'); +} + +/** Counts for a summary line. The unsupported breakdown appears only when something is unsupported, so other summaries are unchanged. */ +export function describeWdioCounts(results: readonly { status: string; gap?: WdioCaseOutcome['gap'] }[]): string { + const count = (status: string) => results.filter((result) => result.status === status).length; + const parts = [`${count('passed')} passed`, `${count('skipped')} skipped`]; + const unsupported = results.filter(({ status }) => status === 'unsupported'); + if (unsupported.length) { + parts.push( + describeDesktopUnsupported({ + unsupported: unsupported.length, + declaredUnsupported: unsupported.filter(({ gap }) => gap === 'unsupported').length, + quarantined: unsupported.filter(({ gap }) => gap === 'quarantine').length, + }), + ); + } + return parts.join(', '); +} + +/** + * A cohort is incomplete when selected work did not finish, or when nothing executed and it is not `optional`. In an + * input lane a run-time skip, or an `unsupported` result with no declared gap, is also incomplete, and `optional` never + * masks it: only a declaration may excuse a case the lane selected. A cohort that is all declared gaps is the caller's + * to judge, because a smoke run executes one story at a time and so cannot require every story to run something. + */ +export function assessWdioCohort(options: { + results: readonly WdioCaseOutcome[]; + selected: number; + lane?: DesktopInputLane; + optional?: boolean; +}): WdioCohortVerdict { + const { results, selected, lane, optional } = options; + const executed = hasExecutedWdioCase(results); + const undeclared = lane + ? results.filter(({ status, gap }) => status === 'skipped' || (status === 'unsupported' && gap === undefined)) + : []; + const reasons: string[] = []; + if (results.length !== selected) reasons.push(`${selected - results.length} of ${selected} selected tests did not finish`); + if (undeclared.length) { + const names = undeclared.slice(0, 5).map(({ storyId, testName }) => (testName === undefined ? storyId : `${storyId} / ${testName}`)); + const more = undeclared.length > names.length ? `, and ${undeclared.length - names.length} more` : ''; + reasons.push( + `the ${lane} lane could not run ${undeclared.length} selected tests that have no declared gap (${names.join(', ')}${more})`, + ); + } + if (!executed && !optional) reasons.push('no selected test executed'); + return { + executed, + incomplete: reasons.length > 0, + ...(reasons.length ? { message: `Required executable story cohort is incomplete: ${reasons.join('; ')}.` } : {}), + }; +} diff --git a/packages/agentic/storybook-desktop/src/testing/runWdioTests.test.ts b/packages/agentic/storybook-desktop/src/testing/runWdioTests.test.ts deleted file mode 100644 index 5860dd8e4f..0000000000 --- a/packages/agentic/storybook-desktop/src/testing/runWdioTests.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { validateDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; -import type { DesktopStoryManifest } from '@fluentui-react-native/desktop-driver'; - -import { missingRequirements, resolveCaseClickMode, selectWdioCases, selectWdioStories } from './runWdioTests.js'; - -describe('WDIO input lane selection', () => { - const manifest: DesktopStoryManifest = { - endpoint: 'win32', - entries: [ - { - id: 'button', - name: 'Button', - packageName: 'components', - sourcePath: 'button.stories.tsx', - tags: ['story'], - title: 'Button', - wdio: { - digest: 'digest', - exportName: 'Default', - testNames: ['adaptive', 'accessible', 'win32-only'], - cases: [ - { name: 'adaptive', options: validateDesktopCaseOptions({ requires: ['click'] }) }, - { name: 'accessible', options: validateDesktopCaseOptions({ requires: ['accessibility-click'] }) }, - { name: 'win32-only', options: validateDesktopCaseOptions({ platforms: ['win32'] }) }, - ], - }, - }, - ], - platformManifestDigest: 'digest', - portablePlanDigest: 'digest', - schemaVersion: 1, - }; - - test('selects by modality and target platform', () => { - expect(selectWdioStories(manifest, { input: 'accessible' }, 'win32').map(({ id }) => id)).toEqual(['button']); - expect(selectWdioCases(manifest.entries[0], { input: 'accessible' }, 'win32')?.map(({ name }) => name)).toEqual([ - 'adaptive', - 'accessible', - 'win32-only', - ]); - expect(selectWdioCases(manifest.entries[0], {}, 'macos')?.map(({ name }) => name)).toEqual(['adaptive', 'accessible']); - expect(selectWdioCases(manifest.entries[0], { input: 'physical' }, 'win32')?.map(({ name }) => name)).toEqual([ - 'adaptive', - 'win32-only', - ]); - expect(selectWdioCases(manifest.entries[0], { input: 'physical', test: 'accessible' }, 'macos')).toEqual([]); - expect(selectWdioCases(manifest.entries[0], { input: 'physical', test: 'win32-only' }, 'macos')).toEqual([]); - }); - - test('rejects legacy callback entries in a lane', () => { - const legacy = { - ...manifest, - entries: [{ ...manifest.entries[0], wdio: { digest: 'legacy', exportName: 'Default', testNames: ['legacy'] } }], - } satisfies DesktopStoryManifest; - expect(() => selectWdioStories(legacy, { input: 'physical' }, 'win32')).toThrow('declare it with desktopIt'); - }); - - test('adapts click requirements to each lane without fallback', () => { - const descriptor = validateDesktopCaseOptions({ requires: ['click'] }); - const accessibleMode = resolveCaseClickMode(descriptor, 'auto', 'accessible'); - const physicalMode = resolveCaseClickMode(descriptor, 'auto', 'physical'); - expect(accessibleMode).toBe('accessibility'); - expect(physicalMode).toBe('physical'); - expect(missingRequirements(descriptor, { accessibilityClick: true, physicalClick: false }, physicalMode)).toEqual(['click']); - expect(missingRequirements(descriptor, { accessibilityClick: false, physicalClick: true }, accessibleMode)).toEqual(['click']); - expect(missingRequirements(descriptor, { accessibilityClick: true }, accessibleMode)).toEqual([]); - }); -}); diff --git a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts index 8ed1c3a371..524bd88ae3 100644 --- a/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts +++ b/packages/agentic/storybook-desktop/src/testing/runWdioTests.ts @@ -3,9 +3,13 @@ import { createHash } from 'node:crypto'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; -import type { DesktopStoryManifest, DesktopStoryManifestEntry } from '@fluentui-react-native/desktop-driver'; -import { matchesDesktopPlatforms } from '@fluentui-react-native/desktop-driver/authoring'; -import type { DesktopArtifact, DesktopInputLane, ResolvedDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; +import type { DesktopStoryManifest } from '@fluentui-react-native/desktop-driver'; +import { + missingDesktopRequirements, + resolveDesktopClickMode, + unmaskedAccessibleFeatures, + type DesktopArtifact, +} from '@fluentui-react-native/desktop-driver/authoring'; import { ArtifactManager, captureDesktopFailureEvidence } from '@fluentui-react-native/desktop-driver/artifacts'; import { connectDesktopWebdriver } from '@fluentui-react-native/desktop-driver/wdio'; import type { DesktopReportWriter } from '@fluentui-react-native/desktop-test/reporting'; @@ -16,6 +20,15 @@ import type { Platforms } from '../config/platforms.js'; import { resolveWdioOptions, type DesktopStorybookWdioOptions } from '../config/wdio.js'; import { NodeDesktopCommandRunner, type DesktopCommandRunner, type PreparedDesktopCommand } from '../cli/commandRunner.js'; import { extractWdioTests } from './extractWdioTests.js'; +import { assessWdioCohort } from './laneVerdict.js'; +import { + csfTestId, + decideWdioSelection, + describeWdioExclusions, + matchesWdioTest, + plannedWdioWork, + type WdioSelectionContext, +} from './selection.js'; import type { WdioWorkerOptions } from './worker.js'; import { formatDesktopStorybookError, writeDesktopStorybookFailure, type DesktopStorybookErrorOutput } from '../../config/diagnostics.cjs'; @@ -35,6 +48,8 @@ export type RunWdioStoryTestsOptions = DesktopStorybookWdioOptions & { config: DesktopStorybookConfig; manifest: DesktopStoryManifest; platform: Platforms; + /** The scope domains and values of this target, for case `scope` selection. */ + scope?: WdioSelectionContext['scope']; targetId: string; url: string; artifactsRoot?: string; @@ -42,6 +57,8 @@ export type RunWdioStoryTestsOptions = DesktopStorybookWdioOptions & { report?: DesktopReportWriter; signal?: AbortSignal; optional?: boolean; + /** The caller already reported the selection and announced the planned work, so this call must not repeat it. */ + announced?: boolean; }; export class WdioStoryTestRunError extends Error { @@ -61,36 +78,13 @@ class MissingRequirementsError extends Error { } } -export function selectWdioStories( - manifest: DesktopStoryManifest, - options: DesktopStorybookWdioOptions, - platform?: Platforms, -): readonly DesktopStoryManifestEntry[] { - return manifest.entries - .filter((entry) => entry.wdio && (!options.story || path.matchesGlob(entry.id, options.story))) - .filter((entry) => !options.tag || entry.tags.includes(options.tag)) - .filter((entry) => { - if (options.input || entry.wdio?.cases) return (selectWdioCases(entry, options, platform)?.length ?? 0) > 0; - return (entry.wdio?.testNames ?? ['default']).some((name) => matchesWdioTest(name, options.test)); - }) - .sort((left, right) => left.id.localeCompare(right.id)); -} - -export function selectWdioCases( - entry: DesktopStoryManifestEntry, - options: DesktopStorybookWdioOptions, - platform?: Platforms, -): NonNullable['cases'] { - if (!entry.wdio?.cases?.length) { - if (!options.input) return undefined; - throw new Error(`Input lane "${options.input}" cannot run legacy wdio callback in "${entry.id}"; declare it with desktopIt.`); +/** The driver cannot show it disabled physical input, so an accessible lane could not keep its guarantee. */ +class UnmaskedLaneError extends Error { + constructor(unmasked: readonly string[]) { + super( + `The accessible input lane needs a driver that reports ${unmasked.join(', ')} disabled, and this one does not. Start it with "driver --input accessible".`, + ); } - return entry.wdio.cases.filter( - ({ name, options: descriptor }) => - matchesWdioTest(name, options.test) && - (!platform || matchesDesktopPlatforms(descriptor.platforms, platform)) && - (!options.input || descriptor.input === 'both' || descriptor.input === options.input), - ); } export async function runWdioStoryTests( @@ -98,31 +92,37 @@ export async function runWdioStoryTests( runner?: DesktopCommandRunner, ): Promise { const settings = resolveWdioOptions(options); - const stories = selectWdioStories(options.manifest, settings, options.platform); - if (stories.length === 0) { - throw new Error('No executable wdio story tests matched the requested selection.'); + const selection = decideWdioSelection(options.manifest, settings, { platform: options.platform, scope: options.scope }); + const planned = plannedWdioWork(selection, settings); + const { platform: byPlatform, scope: byScope, input: byInput } = selection.excluded; + // Reported before any empty-selection error so the exclusions are accounted for. A lane always states them, + // even when nothing was excluded; compatibility mode only when it excluded something. + if (!options.announced && options.report && (settings.input || byPlatform + byScope + byInput > 0)) { + options.report.emit({ kind: 'selection', payload: { excluded: selection.excluded } }); + } + if (selection.stories.length === 0) { + throw new Error(`No executable wdio story tests matched the requested selection${describeWdioExclusions(selection)}.`); } - const cases = stories.flatMap((entry) => { + const cases = selection.stories.flatMap((entry) => { const sourceFile = path.join(options.config.resolvePackage(entry.packageName).root, entry.sourcePath); const extracted = extractWdioTests(fs.readFileSync(sourceFile, 'utf8'), sourceFile).get(entry.wdio!.exportName); if (!extracted || extracted.digest !== entry.wdio!.digest) { throw new Error(`Executable test source changed for ${entry.id}; regenerate the Storybook manifest.`); } - const selectedCases = selectWdioCases(entry, settings, options.platform); + const declared = selection.selected.get(entry.id); return extracted.tests .filter(({ name }) => matchesWdioTest(name ?? 'default', settings.test)) - .map(({ name, code }) => ({ - entry, - code, - testName: name, - descriptor: entry.wdio?.cases?.find((testCase) => testCase.name === name)?.options, - })) - .filter(({ testName }) => { - if (!selectedCases) return !settings.input; - return selectedCases.some(({ name }) => name === testName); + .flatMap(({ name, code }) => { + const decision = name === undefined ? undefined : declared?.get(name); + if (declared && !decision) return []; + return [{ entry, code, testName: name, descriptor: decision?.descriptor, gap: decision?.gap }]; }); }); if (cases.length === 0) throw new Error('No executable wdio tests matched the requested selection.'); + const plannedIds = new Set(planned.map(({ workId }) => workId)); + if (cases.length !== plannedIds.size || cases.some(({ entry, testName }) => !plannedIds.has(csfTestId(entry.id, testName)))) { + throw new Error('The executable tests differ from the selected manifest cases; regenerate the Storybook manifest.'); + } const artifacts = new ArtifactManager( options.artifactsRoot ?? (options.report @@ -135,11 +135,7 @@ export async function runWdioStoryTests( ).href; const results: WdioStoryTestResult[] = []; const executionFailures: unknown[] = []; - for (const { entry, testName } of cases) - options.report?.emit({ - kind: 'work-selected', - payload: { workId: csfTestId(entry.id, testName), label: `${entry.id} / ${testName ?? 'default'}` }, - }); + if (!options.announced) for (const payload of planned) options.report?.emit({ kind: 'work-selected', payload }); const ownedRunner = runner ? undefined : new NodeDesktopCommandRunner({ reporter: options.report }); const commands = runner ?? ownedRunner!; let directory: string | undefined; @@ -151,7 +147,7 @@ export async function runWdioStoryTests( try { fs.mkdirSync(generatedRoot, { recursive: true }); directory = fs.mkdtempSync(path.join(generatedRoot, 'run-')); - for (const [index, { entry, code, testName, descriptor }] of cases.entries()) { + for (const [index, { entry, code, testName, descriptor, gap }] of cases.entries()) { options.signal?.throwIfAborted(); const displayName = testName === undefined ? entry.id : `${entry.id} / ${testName}`; const artifactId = @@ -171,7 +167,6 @@ export async function runWdioStoryTests( const context = { story: entry.id, test: testName ?? 'default', endpoint: options.platform }; options.report?.emit({ kind: 'scope-start', payload: { label: displayName, category: 'worker' } }, testId, context, testId); options.report?.emit({ kind: 'test-start', payload: { testId, title: displayName, attempt: 0 } }, testId, context); - const gap = descriptor && declaredGapFor(descriptor, options.platform, settings.input); if (gap) { result.status = 'unsupported'; result.skipReason = gap.reason; @@ -181,7 +176,15 @@ export async function runWdioStoryTests( options.report?.emit( { kind: 'test-result', - payload: { testId, title: displayName, attempt: 0, status: 'unsupported', durationMs: result.durationMs, reason: gap.reason }, + payload: { + testId, + title: displayName, + attempt: 0, + status: 'unsupported', + durationMs: result.durationMs, + reason: gap.reason, + gap: gap.kind, + }, }, testId, context, @@ -190,7 +193,7 @@ export async function runWdioStoryTests( activeResult = undefined; continue; } - const clickMode = resolveCaseClickMode(descriptor, settings.clickMode, settings.input); + const clickMode = resolveDesktopClickMode(descriptor, settings.clickMode, settings.input); // The supervisor owns the session, so even a crashed or timed-out Node worker cannot strand it. const desktop = await connectDesktopWebdriver({ clickMode, @@ -206,7 +209,13 @@ export async function runWdioStoryTests( if (live.endpoint !== options.platform || live.platformManifestDigest !== options.manifest.platformManifestDigest) { throw new Error('The running Storybook manifest is stale or targets a different platform. Restart storybook driver and the app.'); } - const missing = missingRequirements(descriptor, desktop.browser.capabilities['furn:features'], clickMode); + if (settings.input === 'accessible') { + // A driver that cannot report its physical input disabled fails closed rather than silently running unmasked. + phase = 'lane verification'; + const unmasked = unmaskedAccessibleFeatures(desktop.browser.capabilities['furn:features']); + if (unmasked.length) throw new UnmaskedLaneError(unmasked); + } + const missing = missingDesktopRequirements(descriptor, desktop.browser.capabilities['furn:features'], clickMode); if (missing.length) throw new MissingRequirementsError(missing); phase = 'navigation'; await desktop.openStory(entry.id); @@ -263,6 +272,8 @@ export async function runWdioStoryTests( if (error instanceof MissingRequirementsError) { result.status = 'unsupported'; result.skipReason = error.message; + } else if (error instanceof UnmaskedLaneError) { + failure = error; } else { failure = error; executionFailures.push(error); @@ -298,6 +309,11 @@ export async function runWdioStoryTests( `Could not release the wdio session for ${displayName}; stopping before another test starts.`, ); } + // Every case would fail the same way, so stop here and let the run report it once, against this case. + if (failure instanceof UnmaskedLaneError) { + phase = 'lane verification'; + throw failure; + } result.durationMs = Date.now() - started; results.push(result); options.report?.emit( @@ -348,14 +364,9 @@ export async function runWdioStoryTests( } const failures: unknown[] = infrastructureFailure === undefined ? [] : [infrastructureFailure]; failures.push(...executionFailures); - const allSkipped = results.every(({ status }) => status === 'skipped'); - const laneIncomplete = - !!settings.input && - (!results.some(({ status }) => status === 'passed' || status === 'failed') || - results.some(({ status, gap }) => status === 'skipped' || (status === 'unsupported' && gap === undefined))); - const incomplete = results.length !== cases.length || (allSkipped && !options.optional) || laneIncomplete; - if (incomplete && failures.length === 0) - failures.push(new Error('Required executable story cohort did not execute any tests or left selected work unfinished.')); + const verdict = assessWdioCohort({ results, selected: cases.length, lane: settings.input, optional: options.optional }); + const incomplete = verdict.incomplete; + if (incomplete && failures.length === 0) failures.push(new Error(verdict.message)); try { const reportPath = artifacts.writeMetadata('run', { endpoint: options.platform, @@ -364,9 +375,9 @@ export async function runWdioStoryTests( ? incomplete && executionFailures.length === 0 && infrastructureFailure === undefined ? 'incomplete' : 'failed' - : allSkipped - ? 'skipped' - : 'passed', + : verdict.executed + ? 'passed' + : 'skipped', ...(infrastructureFailure === undefined ? {} : { error: errorMessage(infrastructureFailure) }), tests: results, }); @@ -407,85 +418,6 @@ function isWorkerResult(value: unknown): value is WorkerResult { ); } -export function matchesWdioTest(name: string, pattern?: string): boolean { - return pattern === undefined || path.matchesGlob(name, pattern); -} - -function csfTestId(storyId: string, testName?: string): string { - return `csf-${createHash('sha256') - .update(JSON.stringify([storyId, testName ?? 'default'])) - .digest('hex')}`; -} - -export function resolveCaseClickMode( - descriptor: ResolvedDesktopCaseOptions | undefined, - configured: 'auto' | 'physical' | 'accessibility', - input: DesktopInputLane | undefined, -): 'auto' | 'physical' | 'accessibility' { - if (input === 'accessible') return 'accessibility'; - if (input === 'physical') return 'physical'; - if (descriptor?.input === 'accessible' || descriptor?.requires?.includes('accessibility-click')) return 'accessibility'; - if (descriptor?.input === 'physical') return 'physical'; - return configured; -} - -export function missingRequirements( - descriptor: ResolvedDesktopCaseOptions | undefined, - features: Readonly> | undefined, - clickMode: 'auto' | 'physical' | 'accessibility', -): string[] { - if (!descriptor?.requires?.length) return []; - const missing: string[] = []; - for (const requirement of descriptor.requires) { - const feature = - requirement === 'click' - ? clickMode === 'physical' - ? 'physicalClick' - : clickMode === 'accessibility' - ? 'accessibilityClick' - : undefined - : requirement === 'physical-click' - ? 'physicalClick' - : requirement === 'accessibility-click' - ? 'accessibilityClick' - : requirement === 'accessibility-actions' - ? 'accessibilityActions' - : requirement === 'element-screenshot' - ? 'elementScreenshot' - : requirement === 'set-window-rect' - ? 'setWindowRect' - : requirement; - if (requirement === 'click' && clickMode === 'auto') { - if (!features?.physicalClick && !features?.accessibilityClick) missing.push(requirement); - } else if (feature && features?.[feature] !== true) { - missing.push(requirement); - } - } - return missing; -} - -function declaredGapFor( - descriptor: ResolvedDesktopCaseOptions, - endpoint: Platforms, - input: DesktopInputLane | undefined, -): { kind: 'unsupported' | 'quarantine'; reason: string } | undefined { - if (descriptor.unsupported && matchesDesktopPlatforms(descriptor.unsupported.platforms, endpoint)) { - return { kind: 'unsupported', reason: descriptor.unsupported.reason }; - } - const quarantine = descriptor.quarantine; - if ( - quarantine && - (!quarantine.platforms || matchesDesktopPlatforms(quarantine.platforms, endpoint)) && - (!quarantine.input || quarantine.input === input) - ) { - if (Date.parse(`${quarantine.expires}T00:00:00Z`) < Date.now()) { - throw new Error(`Quarantine ${quarantine.issue} expired on ${quarantine.expires}.`); - } - return { kind: 'quarantine', reason: `${quarantine.issue}: ${quarantine.reason}` }; - } - return undefined; -} - function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } diff --git a/packages/agentic/storybook-desktop/src/testing/selection.test.ts b/packages/agentic/storybook-desktop/src/testing/selection.test.ts new file mode 100644 index 0000000000..847a27062e --- /dev/null +++ b/packages/agentic/storybook-desktop/src/testing/selection.test.ts @@ -0,0 +1,301 @@ +import type { DesktopStoryManifest, DesktopStoryManifestEntry } from '@fluentui-react-native/desktop-driver'; +import { resolveDesktopScopeDomains, validateDesktopCaseOptions } from '@fluentui-react-native/desktop-driver/authoring'; + +import { + csfTestId, + decideWdioSelection, + describeWdioExclusions, + listWdioSelection, + plannedWdioWork, + type WdioSelection, +} from './selection.js'; + +type DeclaredCase = readonly [name: string, options: unknown]; + +function declared(id: string, cases: readonly DeclaredCase[], tags: string[] = ['story']): DesktopStoryManifestEntry { + return { + id, + name: id, + packageName: 'components', + sourcePath: `${id}.stories.tsx`, + tags, + title: id, + wdio: { + digest: 'digest', + exportName: 'Default', + testNames: cases.map(([name]) => name), + cases: cases.map(([name, options]) => ({ name, options: validateDesktopCaseOptions(options) })), + }, + }; +} + +function legacy(id: string, testNames?: string[]): DesktopStoryManifestEntry { + return { + id, + name: id, + packageName: 'components', + sourcePath: `${id}.stories.tsx`, + tags: ['story'], + title: id, + wdio: { digest: 'legacy', exportName: 'Default', ...(testNames ? { testNames } : {}) }, + }; +} + +function manifestOf(...entries: DesktopStoryManifestEntry[]): DesktopStoryManifest { + return { endpoint: 'win32', entries, platformManifestDigest: 'digest', portablePlanDigest: 'digest', schemaVersion: 1 }; +} + +const selectedNames = (selection: WdioSelection, id = 'button') => [...(selection.selected.get(id)?.keys() ?? [])]; + +const button = declared('button', [ + ['adaptive', { requires: ['click'] }], + ['accessible', { requires: ['accessibility-click'] }], + ['win32-only', { platforms: ['win32'] }], +]); + +describe('WDIO selection', () => { + test('selects by input lane and target platform', () => { + const manifest = manifestOf(button); + expect(selectedNames(decideWdioSelection(manifest, { input: 'accessible' }, { platform: 'win32' }))).toEqual([ + 'adaptive', + 'accessible', + 'win32-only', + ]); + expect(selectedNames(decideWdioSelection(manifest, {}, { platform: 'macos' }))).toEqual(['adaptive', 'accessible']); + expect(selectedNames(decideWdioSelection(manifest, { input: 'physical' }, { platform: 'win32' }))).toEqual(['adaptive', 'win32-only']); + }); + + test('records every decision with its first exclusion reason', () => { + const selection = decideWdioSelection(manifestOf(button), { input: 'physical' }, { platform: 'macos' }); + expect(selection.decisions.map(({ name, excluded }) => [name, excluded])).toEqual([ + ['adaptive', undefined], + ['accessible', 'input'], + ['win32-only', 'platform'], + ]); + expect(selection.excluded).toEqual({ platform: 1, scope: 0, input: 1 }); + }); + + test('counts exclusions only among the cases the user filters selected', () => { + const manifest = manifestOf(button, declared('other', [['unrelated', { platforms: ['win32'] }]])); + const byTest = decideWdioSelection(manifest, { input: 'physical', test: 'accessible' }, { platform: 'macos' }); + expect(byTest.stories).toEqual([]); + expect(byTest.excluded).toEqual({ platform: 0, scope: 0, input: 1 }); + const byStory = decideWdioSelection(manifest, { input: 'physical', story: 'button', test: 'win32-only' }, { platform: 'macos' }); + expect(byStory.excluded).toEqual({ platform: 1, scope: 0, input: 0 }); + expect(describeWdioExclusions(byStory)).toBe(' (excluded 1 by platform, 0 by scope, 0 by input)'); + expect(describeWdioExclusions(decideWdioSelection(manifest, {}, { platform: 'win32' }))).toBe(''); + }); + + test('excludes by platform, then scope, then input', () => { + const layered = manifestOf( + declared('layered', [['all', { platforms: ['win32'], scope: { 'rn.renderer': ['paper'] }, requires: ['keyboard'] }]]), + ); + const decide = (platform: 'macos' | 'win32', renderer: string) => + decideWdioSelection( + layered, + { input: 'accessible' }, + { platform, scope: { domains: resolveDesktopScopeDomains(), values: { 'rn.renderer': renderer } } }, + ); + expect(decide('macos', 'fabric').excluded).toEqual({ platform: 1, scope: 0, input: 0 }); + expect(decide('win32', 'fabric').excluded).toEqual({ platform: 0, scope: 1, input: 0 }); + expect(decide('win32', 'paper').excluded).toEqual({ platform: 0, scope: 0, input: 1 }); + }); + + test('scopes cases by a value the target holds, including custom domains', () => { + const domains = resolveDesktopScopeDomains({ 'product.flavor': ['alpha', 'beta'] }); + const scoped = manifestOf( + declared('scoped', [ + ['paper-only', { scope: { 'rn.renderer': ['paper'] } }], + ['beta-only', { scope: { 'product.flavor': ['beta'] } }], + ['both-keys', { scope: { 'rn.renderer': ['paper', 'fabric'], 'product.flavor': ['alpha'] } }], + ]), + ); + const values = (renderer: string, flavor: string) => ({ domains, values: { 'rn.renderer': renderer, 'product.flavor': flavor } }); + expect(selectedNames(decideWdioSelection(scoped, {}, { platform: 'win32', scope: values('paper', 'beta') }), 'scoped')).toEqual([ + 'paper-only', + 'beta-only', + ]); + expect(selectedNames(decideWdioSelection(scoped, {}, { platform: 'win32', scope: values('fabric', 'alpha') }), 'scoped')).toEqual([ + 'both-keys', + ]); + }); + + test('treats a scope key the target does not supply as a configuration error rather than an exclusion', () => { + const scoped = manifestOf(declared('scoped', [['paper-only', { scope: { 'rn.renderer': ['paper'] } }]])); + expect(() => decideWdioSelection(scoped, {}, { platform: 'win32' })).toThrow('is not supplied by this target'); + const custom = manifestOf(declared('scoped', [['flavored', { scope: { 'product.flavor': ['beta'] } }]])); + expect(() => + decideWdioSelection(custom, {}, { platform: 'win32', scope: { domains: resolveDesktopScopeDomains(), values: {} } }), + ).toThrow('unknown scope key "product.flavor"'); + const outside = manifestOf(declared('scoped', [['bogus', { scope: { 'rn.renderer': ['bogus'] } }]])); + expect(() => + decideWdioSelection( + outside, + {}, + { platform: 'win32', scope: { domains: resolveDesktopScopeDomains(), values: { 'rn.renderer': 'paper' } } }, + ), + ).toThrow('is not one of: fabric, paper'); + }); + + test('does not evaluate the scope of a case another platform already excluded', () => { + const scoped = manifestOf(declared('scoped', [['win32-paper', { platforms: ['win32'], scope: { 'rn.renderer': ['paper'] } }]])); + expect(decideWdioSelection(scoped, {}, { platform: 'macos' }).excluded).toEqual({ platform: 1, scope: 0, input: 0 }); + }); + + test('filters by story glob and tag and orders stories by id', () => { + const manifest = manifestOf( + declared('zeta', [['a', {}]], ['wide']), + declared('alpha', [['a', {}]], ['wide']), + declared('middle', [['a', {}]], ['narrow']), + ); + expect(decideWdioSelection(manifest, {}, { platform: 'win32' }).stories.map(({ id }) => id)).toEqual(['alpha', 'middle', 'zeta']); + expect(decideWdioSelection(manifest, { tag: 'wide' }, { platform: 'win32' }).stories.map(({ id }) => id)).toEqual(['alpha', 'zeta']); + expect(decideWdioSelection(manifest, { story: 'm*' }, { platform: 'win32' }).stories.map(({ id }) => id)).toEqual(['middle']); + }); + + describe('legacy callbacks', () => { + test('run in compatibility mode when a name matches the test filter', () => { + const manifest = manifestOf(legacy('plain'), legacy('named', ['first', 'second']), declared('button', [['a', {}]])); + expect(decideWdioSelection(manifest, {}, { platform: 'win32' }).stories.map(({ id }) => id)).toEqual(['button', 'named', 'plain']); + expect(decideWdioSelection(manifest, { test: 'sec*' }, { platform: 'win32' }).stories.map(({ id }) => id)).toEqual(['named']); + expect(decideWdioSelection(manifest, { test: 'nothing' }, { platform: 'win32' }).stories).toEqual([]); + }); + + test('are rejected together in a lane so one pass migrates them', () => { + const manifest = manifestOf(legacy('b-legacy'), legacy('a-legacy', ['first']), declared('button', [['a', {}]])); + expect(() => decideWdioSelection(manifest, { input: 'physical' }, { platform: 'win32' })).toThrow( + 'Input lane "physical" cannot run legacy wdio callbacks; declare them with desktopIt: a-legacy, b-legacy.', + ); + }); + + test('are ignored in a lane when the test filter names nothing they define, and summarized past ten', () => { + const many = Array.from({ length: 12 }, (_, index) => legacy(`legacy-${String(index).padStart(2, '0')}`)); + expect(() => decideWdioSelection(manifestOf(...many), { input: 'accessible' }, { platform: 'win32' })).toThrow( + /legacy-00, legacy-01.*legacy-09, and 2 more\.$/, + ); + expect(decideWdioSelection(manifestOf(...many), { input: 'accessible', test: 'absent' }, { platform: 'win32' }).stories).toEqual([]); + }); + }); + + describe('declared gaps', () => { + const gapped = manifestOf( + declared('gapped', [ + ['unsupported', { unsupported: { platforms: ['win32'], reason: 'No host support.' } }], + ['quarantined', { quarantine: { issue: '#9', reason: 'flaky', expires: '2026-01-01' } }], + ['physical-quarantine', { quarantine: { issue: '#10', reason: 'drifts', expires: '2026-01-01', input: 'physical' } }], + ]), + ); + const before = Date.parse('2025-12-31T00:00:00Z'); + + test('are resolved at selection time for the cases that run', () => { + const win32 = decideWdioSelection(gapped, { input: 'physical' }, { platform: 'win32', now: before }); + expect([...win32.selected.get('gapped')!.values()].map(({ name, gap }) => [name, gap])).toEqual([ + ['unsupported', { kind: 'unsupported', reason: 'No host support.' }], + ['quarantined', { kind: 'quarantine', reason: '#9: flaky' }], + ['physical-quarantine', { kind: 'quarantine', reason: '#10: drifts' }], + ]); + const macos = decideWdioSelection(gapped, { input: 'accessible' }, { platform: 'macos', now: before }); + expect([...macos.selected.get('gapped')!.values()].map(({ name, gap }) => [name, gap?.kind])).toEqual([ + ['unsupported', undefined], + ['quarantined', 'quarantine'], + ['physical-quarantine', undefined], + ]); + }); + + test('stop the run before anything executes once a quarantine expires', () => { + expect(() => decideWdioSelection(gapped, {}, { platform: 'win32', now: Date.parse('2026-06-01T00:00:00Z') })).toThrow( + 'Quarantine #9 expired on 2026-01-01.', + ); + }); + + test('are not evaluated for a case the lane already excluded', () => { + const excludedByLane = manifestOf( + declared('gapped', [ + ['accessible', { requires: ['accessibility-click'], quarantine: { issue: '#9', reason: 'old', expires: '2020-01-01' } }], + ]), + ); + expect(decideWdioSelection(excludedByLane, { input: 'physical' }, { platform: 'win32' }).excluded.input).toBe(1); + }); + }); + + test('plans the work each selected case announces, legacy included', () => { + const manifest = manifestOf( + declared('button', [ + ['adaptive', {}], + ['physical', { requires: ['keyboard'] }], + ]), + legacy('named', ['x', 'y']), + ); + const planned = plannedWdioWork(decideWdioSelection(manifest, {}, { platform: 'win32' }), {}); + expect(planned).toEqual([ + { workId: csfTestId('button', 'adaptive'), label: 'button / adaptive' }, + { workId: csfTestId('button', 'physical'), label: 'button / physical' }, + { workId: csfTestId('named', 'x'), label: 'named / x' }, + { workId: csfTestId('named', 'y'), label: 'named / y' }, + ]); + const filtered = plannedWdioWork(decideWdioSelection(manifest, { test: 'y' }, { platform: 'win32' }), { test: 'y' }); + expect(filtered.map(({ label }) => label)).toEqual(['named / y']); + const lane = plannedWdioWork(decideWdioSelection(manifestOf(manifest.entries[0]), { input: 'accessible' }, { platform: 'win32' }), {}); + expect(lane.map(({ label }) => label)).toEqual(['button / adaptive']); + }); + + test('names a legacy callback without a declared name "default"', () => { + const planned = plannedWdioWork(decideWdioSelection(manifestOf(legacy('plain')), {}, { platform: 'win32' }), {}); + expect(planned).toEqual([{ workId: csfTestId('plain', 'default'), label: 'plain / default' }]); + expect(csfTestId('plain')).toBe(csfTestId('plain', 'default')); + }); + + describe('listing', () => { + const listed = manifestOf( + declared('button', [ + ['adaptive', { requires: ['click'] }], + ['physical', { requires: ['keyboard'], platforms: ['win32'] }], + ['accessible', { requires: ['accessibility-click'], unsupported: { platforms: ['win32'], reason: 'No host support.' } }], + ]), + legacy('named', ['first', 'second']), + legacy('plain'), + declared('only-physical', [['physical', { requires: ['keyboard'] }]]), + ); + + test('shows the selected cases with their gaps and the excluded cases with their first reason', () => { + const selection = decideWdioSelection(listed, { input: 'accessible', story: 'button' }, { platform: 'win32' }); + expect(selection.stories.map(({ id }) => id)).toEqual(['button']); + expect(listWdioSelection(selection, {})).toEqual([ + { + id: 'button', + sourcePath: 'button.stories.tsx', + tests: [ + { name: 'adaptive', input: 'both', inputSource: 'derived', requires: ['click'] }, + { + name: 'accessible', + input: 'accessible', + inputSource: 'derived', + requires: ['accessibility-click'], + gap: { kind: 'unsupported', reason: 'No host support.' }, + }, + ], + excluded: [ + { name: 'physical', input: 'physical', inputSource: 'derived', requires: ['keyboard'], platforms: ['win32'], reason: 'input' }, + ], + }, + ]); + }); + + test('lists a legacy callback by its names, or alone when it names none', () => { + const selection = decideWdioSelection(listed, { story: '{named,plain}', test: 'sec*' }, { platform: 'macos' }); + expect(listWdioSelection(selection, { test: 'sec*' })).toEqual([{ id: 'named', sourcePath: 'named.stories.tsx', tests: ['second'] }]); + expect(listWdioSelection(decideWdioSelection(listed, { story: 'plain' }, { platform: 'macos' }), {})).toEqual([ + { id: 'plain', sourcePath: 'plain.stories.tsx' }, + ]); + }); + + test('omits excluded when every matching case runs', () => { + const [story] = listWdioSelection(decideWdioSelection(listed, { story: 'only-physical' }, { platform: 'win32' }), {}); + expect(story).toEqual({ + id: 'only-physical', + sourcePath: 'only-physical.stories.tsx', + tests: [{ name: 'physical', input: 'physical', inputSource: 'derived', requires: ['keyboard'] }], + }); + }); + }); +}); diff --git a/packages/agentic/storybook-desktop/src/testing/selection.ts b/packages/agentic/storybook-desktop/src/testing/selection.ts new file mode 100644 index 0000000000..e4ae0f18a5 --- /dev/null +++ b/packages/agentic/storybook-desktop/src/testing/selection.ts @@ -0,0 +1,194 @@ +import { createHash } from 'node:crypto'; +import path from 'node:path'; + +import type { DesktopStoryManifest, DesktopStoryManifestEntry } from '@fluentui-react-native/desktop-driver'; +import { decideDesktopCase, resolveDesktopScopeDomains } from '@fluentui-react-native/desktop-driver/authoring'; +import type { + DesktopCaseTarget, + DesktopDeclaredGap, + DesktopScopeDomains, + DesktopScopeValues, + ResolvedDesktopCaseOptions, +} from '@fluentui-react-native/desktop-driver/authoring'; +import type { SelectionExclusions } from '@fluentui-react-native/desktop-test/reporting'; + +import type { Platforms } from '../config/platforms.js'; +import type { DesktopStorybookWdioOptions } from '../config/wdio.js'; + +export type WdioSelectionContext = { + platform: Platforms; + /** The domains and the values this target holds. Without it only the built-in domains exist and no key is supplied. */ + scope?: { domains: DesktopScopeDomains; values: DesktopScopeValues }; + /** Clock for quarantine expiry. */ + now?: number; +}; + +export type WdioDeclaredGap = DesktopDeclaredGap; + +/** One declared case that matched the user's filters. A case with no `excluded` reason is part of the run. */ +export type WdioCaseDecision = { + storyId: string; + name: string; + descriptor: ResolvedDesktopCaseOptions; + /** The first reason, in platform, scope, input order, that this target or lane does not run the case. */ + excluded?: keyof SelectionExclusions; + /** Declared `unsupported` or quarantine; the case is selected but not executed. */ + gap?: WdioDeclaredGap; +}; + +export type WdioSelection = { + /** Stories with at least one selected case, or a legacy callback that compatibility mode may run. */ + stories: readonly DesktopStoryManifestEntry[]; + /** Every declared case that matched the user's filters, in story and source order. */ + decisions: readonly WdioCaseDecision[]; + /** The selected declared cases by story and name. Legacy callbacks are absent. */ + selected: ReadonlyMap>; + /** Only cases that matched the user's filters count, so a narrow `--story` is not blamed for what it never asked for. */ + excluded: SelectionExclusions; +}; + +/** + * Decides every case before any session exists. Legacy callbacks run in compatibility mode and are an error in a lane, + * where each is listed so they can be migrated in one pass. Declared gaps are resolved here too, so an expired + * quarantine stops the run before anything executes. + */ +export function decideWdioSelection( + manifest: DesktopStoryManifest, + options: DesktopStorybookWdioOptions, + context: WdioSelectionContext, +): WdioSelection { + const domains = context.scope?.domains ?? resolveDesktopScopeDomains(); + const target: DesktopCaseTarget = { + endpoint: context.platform, + ...(options.input ? { lane: options.input } : {}), + domains, + values: context.scope?.values ?? {}, + now: context.now, + }; + const stories: DesktopStoryManifestEntry[] = []; + const decisions: WdioCaseDecision[] = []; + const selected = new Map>(); + const excluded: SelectionExclusions = { platform: 0, scope: 0, input: 0 }; + const legacy: string[] = []; + const entries = manifest.entries + .filter((entry) => entry.wdio && (!options.story || path.matchesGlob(entry.id, options.story))) + .filter((entry) => !options.tag || entry.tags.includes(options.tag)) + .sort((left, right) => left.id.localeCompare(right.id)); + for (const entry of entries) { + const cases = entry.wdio!.cases; + if (!cases?.length) { + if (!(entry.wdio!.testNames ?? ['default']).some((name) => matchesWdioTest(name, options.test))) continue; + if (options.input) legacy.push(entry.id); + else stories.push(entry); + continue; + } + const chosen = new Map(); + for (const { name, options: descriptor } of cases) { + if (!matchesWdioTest(name, options.test)) continue; + const outcome = decideDesktopCase(descriptor, target, `${entry.id} / ${name}`); + const decision: WdioCaseDecision = { storyId: entry.id, name, descriptor }; + if (outcome.kind === 'excluded') { + decision.excluded = outcome.reason; + excluded[outcome.reason]++; + } else { + if (outcome.kind === 'gap') decision.gap = outcome.gap; + chosen.set(name, decision); + } + decisions.push(decision); + } + if (chosen.size) { + stories.push(entry); + selected.set(entry.id, chosen); + } + } + if (legacy.length) { + const shown = legacy.slice(0, 10).join(', '); + const more = legacy.length > 10 ? `, and ${legacy.length - 10} more` : ''; + throw new Error(`Input lane "${options.input}" cannot run legacy wdio callbacks; declare them with desktopIt: ${shown}${more}.`); + } + return { stories, decisions, selected, excluded }; +} + +/** The work a selection announces before launch, so work that never starts is still accounted for. */ +export function plannedWdioWork( + selection: WdioSelection, + options: Pick, +): readonly { workId: string; label: string }[] { + return selection.stories.flatMap((entry) => { + const declared = selection.selected.get(entry.id); + const names = declared + ? [...declared.keys()] + : (entry.wdio?.testNames ?? ['default']).filter((name) => matchesWdioTest(name, options.test)); + return names.map((name) => ({ workId: csfTestId(entry.id, name), label: `${entry.id} / ${name}` })); + }); +} + +/** One declared case in a `--list` result. */ +export type WdioListedCase = Pick & { + name: string; + requires: NonNullable; + /** Selected but not executed: a declared `unsupported` entry or an active quarantine. */ + gap?: WdioDeclaredGap; +}; + +export type WdioListedStory = { + id: string; + sourcePath: string; + /** The names of a legacy callback, or the declared cases this target and lane run. */ + tests?: readonly string[] | readonly WdioListedCase[]; + /** Declared cases that matched the filters but do not run, each with the first reason it was excluded. */ + excluded?: readonly (WdioListedCase & { reason: keyof SelectionExclusions })[]; +}; + +/** What a selection would run, and why each other declared case that matched the filters would not. */ +export function listWdioSelection(selection: WdioSelection, options: Pick): WdioListedStory[] { + const decided = new Map(); + for (const decision of selection.decisions) { + const story = decided.get(decision.storyId); + if (story) story.push(decision); + else decided.set(decision.storyId, [decision]); + } + return selection.stories.map(({ id, sourcePath, wdio }) => { + const decisions = decided.get(id); + if (!selection.selected.has(id)) { + const names = wdio?.testNames?.filter((name) => matchesWdioTest(name, options.test)); + return { id, sourcePath, ...(names ? { tests: names } : {}) }; + } + const excluded = decisions!.flatMap((decision) => (decision.excluded ? [{ ...listedCase(decision), reason: decision.excluded }] : [])); + return { + id, + sourcePath, + tests: decisions!.filter(({ excluded }) => !excluded).map(listedCase), + ...(excluded.length ? { excluded } : {}), + }; + }); +} + +/** Names what a selection left out, so an empty selection explains itself. Empty when nothing was excluded. */ +export function describeWdioExclusions({ excluded }: Pick): string { + const { platform, scope, input } = excluded; + return platform + scope + input > 0 ? ` (excluded ${platform} by platform, ${scope} by scope, ${input} by input)` : ''; +} + +export function matchesWdioTest(name: string, pattern?: string): boolean { + return pattern === undefined || path.matchesGlob(name, pattern); +} + +export function csfTestId(storyId: string, testName?: string): string { + return `csf-${createHash('sha256') + .update(JSON.stringify([storyId, testName ?? 'default'])) + .digest('hex')}`; +} + +function listedCase({ name, descriptor, gap }: WdioCaseDecision): WdioListedCase { + const { input, inputSource, platforms, scope, requires = [] } = descriptor; + return { + name, + input, + inputSource, + requires, + ...(platforms ? { platforms } : {}), + ...(scope ? { scope } : {}), + ...(gap ? { gap } : {}), + }; +} diff --git a/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs b/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs index a8bae915fc..3736a9666e 100644 --- a/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs +++ b/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs @@ -25,6 +25,7 @@ async function main() { path.join(root, 'fixture.stories.ts'), ` import 'react-native-must-never-load-in-node'; + import { desktopIt } from '@fluentui-react-native/storybook-desktop/testing'; export default { title: 'Fixture/Button' }; export const Pass = { tags: ['grouped', 'desktop-e2e'], @@ -88,6 +89,17 @@ async function main() { await expect(await browser.$('~button-primary')).toBeEnabled(); }, }}; + export const Lane = { tags: ['desktop-e2e'], wdio: [ + desktopIt('accessible only', { requires: ['accessibility-click'] }, async ({ browser, expect }) => { + await expect(await browser.$('~button-primary')).toBeEnabled(); + }), + desktopIt('either lane', async ({ browser, expect }) => { + await expect(await browser.$('~button-primary')).toBeEnabled(); + }), + desktopIt('physical only', { requires: ['physical-click'] }, async ({ browser, expect }) => { + await expect(await browser.$('~button-primary')).toBeEnabled(); + }), + ]}; `, ); const storyPackage = { name: 'inline-fixture', root, manifest: { name: 'inline-fixture' }, storyPatterns: ['*.stories.ts'] }; @@ -326,6 +338,46 @@ async function main() { } finally { await restrictedButtonHarness.close(); } + const laneResults = (results) => results.map(({ testName, status }) => ({ testName, status })); + // The accessible lane trusts the driver's own report, so a driver that does not show physical input disabled fails closed. + const unmaskedReport = new DesktopRunReporter({ + artifactsRoot: path.join(root, 'lane-reports'), + lane: { input: 'accessible', source: 'option', masking: 'host-masked', policyEnforced: false }, + summary: false, + }); + let unmaskedLane; + try { + await run('lane', { input: 'accessible', report: unmaskedReport }); + throw new Error('An accessible lane must not run against a driver that reports physical input enabled.'); + } catch (error) { + if (!(error instanceof Error) || !(error instanceof WdioStoryTestRunError) || !('results' in error) || !Array.isArray(error.results)) + throw error; + unmaskedLane = { results: laneResults(error.results), message: error.message }; + } + const unmaskedLaneResult = await unmaskedReport.finish(); + const maskedHarness = await createDesktopDriverStoryHarness(manifest, { + features: { keyboard: false, physicalClick: false, wheel: false }, + }); + let accessibleLane; + try { + accessibleLane = laneResults( + await run('lane', { input: 'accessible', targetId: maskedHarness.target.id, url: maskedHarness.server.url }), + ); + } finally { + await maskedHarness.close(); + } + const physicalLane = laneResults(await run('lane', { input: 'physical' })); + // Without a lane a declared physical case must not pin physical click: a host that masks it reports the case unsupported + // rather than failing to create the session, as the skip it replaces did not fail. + const unmaskedPhysicalHarness = await createDesktopDriverStoryHarness(manifest, { features: { physicalClick: false } }); + let compatibilityLane; + try { + compatibilityLane = laneResults( + await run('lane', { targetId: unmaskedPhysicalHarness.target.id, url: unmaskedPhysicalHarness.server.url }), + ); + } finally { + await unmaskedPhysicalHarness.close(); + } process.stdout.write( '\nWDIO_RESULT ' + JSON.stringify({ @@ -338,6 +390,18 @@ async function main() { smokeFailureLogged, button: button.map(({ testName, status }) => ({ testName, status })), restrictedButton: restrictedButton.map(({ testName, status }) => ({ testName, status })), + lane: { + accessible: accessibleLane, + physical: physicalLane, + compatibility: compatibilityLane, + unmasked: { + ...unmaskedLane, + status: unmaskedLaneResult.status, + failed: unmaskedLaneResult.counts.failed, + excluded: unmaskedLaneResult.selection?.excluded, + reported: unmaskedLaneResult.failures.map(({ error }) => error.message), + }, + }, platforms, namedFailureResults: namedFailureResults.map(({ testName, status }) => ({ testName, status })), namedTimeout, diff --git a/packages/agentic/storybook-desktop/src/testing/wdioRunner.test.ts b/packages/agentic/storybook-desktop/src/testing/wdioRunner.test.ts index 01a741bdfd..c69d9b8c56 100644 --- a/packages/agentic/storybook-desktop/src/testing/wdioRunner.test.ts +++ b/packages/agentic/storybook-desktop/src/testing/wdioRunner.test.ts @@ -23,6 +23,7 @@ test('executes isolated inline code and releases sessions after assertion failur !diagnostics.includes('wdio "fixture-button--fail" during execution') || !diagnostics.includes('wdio "fixture-button--navigation-failure" during navigation') || !diagnostics.includes('Storybook context did not settle.') || + !diagnostics.includes('wdio "fixture-button--lane / accessible only" during lane verification') || diagnostics.includes('Navigation refused before dispatch') || output.includes('[storybook] OK wdio "fixture-button--exit-zero"') ) { @@ -55,6 +56,30 @@ test('executes isolated inline code and releases sessions after assertion failur { testName: 'exposes enabled button semantics', status: 'passed' }, { testName: 'supports native pointer activation', status: 'skipped' }, ], + lane: { + accessible: [ + { testName: 'accessible only', status: 'passed' }, + { testName: 'either lane', status: 'passed' }, + ], + physical: [ + { testName: 'either lane', status: 'passed' }, + { testName: 'physical only', status: 'passed' }, + ], + // No lane, and a host that masks physical click: the physical case is unsupported, not a failed session. + compatibility: [ + { testName: 'accessible only', status: 'passed' }, + { testName: 'either lane', status: 'passed' }, + { testName: 'physical only', status: 'unsupported' }, + ], + // A driver that does not report physical input disabled stops the lane at its first case, reported once. + unmasked: { + results: [{ testName: 'accessible only', status: 'failed' }], + status: 'failed', + failed: 1, + excluded: { platform: 0, scope: 0, input: 1 }, + reported: [expect.stringContaining('driver --input accessible')], + }, + }, platforms: Object.fromEntries( ['macos', 'windows', 'win32'].map((platform) => [ platform,