From e42cf26c5ae3a06cf0084af3dfc908a324706c71 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 15 Oct 2026 14:41:01 +0200 Subject: [PATCH] feat: show expected/received diffs for failed assertions The device runner now runs failed test errors through processError from @vitest/utils/error, as Vitest's runner does, and sends the resulting diff in a new optional `diff` field on SerializedError. Merged soft assertion errors keep each failure's diff next to its message, and the Jest reporter prints the diff between the message and the code frame. --- .../version-plan-1792067894893.md | 8 ++ packages/bridge/src/shared/test-runner.ts | 2 + .../__tests__/format-harness-error.test.ts | 20 ++++ packages/jest/src/format-harness-error.ts | 5 + packages/runtime/package.json | 1 + .../src/__tests__/assertion-diff.test.ts | 111 ++++++++++++++++++ packages/runtime/src/expect/diff.ts | 24 ++++ packages/runtime/src/expect/errors.ts | 12 +- packages/runtime/src/runner/errors.ts | 4 + pnpm-lock.yaml | 3 + 10 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 .nx/version-plans/version-plan-1792067894893.md create mode 100644 packages/runtime/src/__tests__/assertion-diff.test.ts create mode 100644 packages/runtime/src/expect/diff.ts diff --git a/.nx/version-plans/version-plan-1792067894893.md b/.nx/version-plans/version-plan-1792067894893.md new file mode 100644 index 00000000..c8223cb6 --- /dev/null +++ b/.nx/version-plans/version-plan-1792067894893.md @@ -0,0 +1,8 @@ +--- +__default__: minor +--- + +Failed assertions now print a `- Expected` / `+ Received` diff below the error +message, the same way Vitest shows them. This works for regular assertions and +for every failed `expect.soft` call. Plugins can read the diff from the new +optional `diff` field on test errors. diff --git a/packages/bridge/src/shared/test-runner.ts b/packages/bridge/src/shared/test-runner.ts index 4ba4c060..f075c4c4 100644 --- a/packages/bridge/src/shared/test-runner.ts +++ b/packages/bridge/src/shared/test-runner.ts @@ -14,6 +14,8 @@ export type TestResultStatus = 'passed' | 'failed' | 'skipped' | 'todo'; export type SerializedError = { name: string; message: string; + // Expected/received diff of a failed assertion, formatted like Vitest's. + diff?: string; codeFrame?: CodeFrame; diagnostics?: { pendingPromises?: { diff --git a/packages/jest/src/__tests__/format-harness-error.test.ts b/packages/jest/src/__tests__/format-harness-error.test.ts index e6fd8506..1a90914a 100644 --- a/packages/jest/src/__tests__/format-harness-error.test.ts +++ b/packages/jest/src/__tests__/format-harness-error.test.ts @@ -29,4 +29,24 @@ describe('formatHarnessErrorMessage', () => { expect(message).toContain('Promise #7, created 10ms after test start:'); expect(message).toContain(' Error: Promise created'); }); + + it('prints the assertion diff between the message and the code frame', () => { + const message = formatHarnessErrorMessage({ + name: 'AssertionError', + message: 'expected 1 to be 2 // Object.is equality', + diff: '- Expected\n+ Received\n\n- 2\n+ 1', + codeFrame: { + content: '> 3 | expect(1).toBe(2);', + fileName: 'example.harness.ts', + }, + }); + + expect(message).toBe( + [ + 'expected 1 to be 2 // Object.is equality', + '- Expected\n+ Received\n\n- 2\n+ 1', + '> 3 | expect(1).toBe(2);', + ].join('\n\n'), + ); + }); }); diff --git a/packages/jest/src/format-harness-error.ts b/packages/jest/src/format-harness-error.ts index ca9c32ac..d5aa68d8 100644 --- a/packages/jest/src/format-harness-error.ts +++ b/packages/jest/src/format-harness-error.ts @@ -54,6 +54,11 @@ export const formatHarnessErrorMessage = ( } const parts = [error.message]; + + if (error.diff) { + parts.push(error.diff); + } + const pendingPromiseDetails = error.diagnostics?.pendingPromises ? formatPendingPromises( error.diagnostics.pendingPromises, diff --git a/packages/runtime/package.json b/packages/runtime/package.json index 96ed35d4..ebde4bb0 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -43,6 +43,7 @@ "@react-native-harness/bridge": "workspace:*", "@vitest/expect": "4.0.16", "@vitest/spy": "4.0.16", + "@vitest/utils": "4.0.16", "chai": "^6.2.2", "event-target-shim": "^6.0.2", "react-native-url-polyfill": "^3.0.0", diff --git a/packages/runtime/src/__tests__/assertion-diff.test.ts b/packages/runtime/src/__tests__/assertion-diff.test.ts new file mode 100644 index 00000000..a3529ed8 --- /dev/null +++ b/packages/runtime/src/__tests__/assertion-diff.test.ts @@ -0,0 +1,111 @@ +import { stripVTControlCharacters } from 'node:util'; +import type { SerializedError } from '@react-native-harness/bridge'; +import { describe, expect, it, vi } from 'vitest'; +import { + describe as harnessDescribe, + getTestCollector, + it as harnessIt, +} from '../collector/index.js'; +import { expect as harnessExpect } from '../expect/index.js'; +import { getTestRunner } from '../runner/index.js'; + +vi.mock('../symbolicate.js', async () => { + return { + getCodeFrame: vi.fn(async () => null), + }; +}); + +const runFailingTest = async ( + fn: () => void, +): Promise<{ result?: SerializedError; event?: SerializedError }> => { + const collector = getTestCollector(); + const runner = getTestRunner(); + let event: SerializedError | undefined; + + runner.events.addListener((runnerEvent) => { + if (runnerEvent.type === 'test-finished') { + event = runnerEvent.error; + } + }); + + try { + const collection = await collector.collect(() => { + harnessDescribe('Diff Suite', () => { + harnessIt('fails', fn); + }); + }, 'runtime/assertion-diff.test.ts'); + + const result = await runner.run({ + testSuite: collection.testSuite, + testFilePath: 'runtime/assertion-diff.test.ts', + runner: 'ios', + }); + + expect(result.suites[0].tests[0].status).toBe('failed'); + + return { result: result.suites[0].tests[0].error, event }; + } finally { + collector.dispose(); + runner.dispose(); + } +}; + +const strip = (value: string | undefined) => + value === undefined ? undefined : stripVTControlCharacters(value); + +// expect.soft is installed at runtime but missing from @vitest/expect's types. +const softExpect = (harnessExpect as unknown as { soft: typeof harnessExpect }) + .soft; + +describe('assertion diffs', () => { + it('serializes the expected/received diff of a failed assertion', async () => { + const { result, event } = await runFailingTest(() => { + harnessExpect({ name: 'Nitro', year: 2018 }).toEqual({ + name: 'Nitro', + year: 2019, + }); + }); + + expect(result?.message).toContain('to deeply equal'); + expect(strip(result?.diff)).toBe( + [ + '- Expected', + '+ Received', + '', + ' {', + ' "name": "Nitro",', + '- "year": 2019,', + '+ "year": 2018,', + ' }', + ].join('\n'), + ); + expect(event?.diff).toBe(result?.diff); + }); + + it('does not add a diff to errors without expected and actual values', async () => { + const { result } = await runFailingTest(() => { + throw new Error('plain failure'); + }); + + expect(result?.message).toBe('plain failure'); + expect(result?.diff).toBeUndefined(); + }); + + it('keeps the diff of every failed soft assertion', async () => { + const { result } = await runFailingTest(() => { + softExpect(1).toBe(2); + softExpect('a').toBe('b'); + }); + + expect(strip(result?.message)).toBe( + [ + 'Soft assertion failures:', + 'AssertionError: expected 1 to be 2 // Object.is equality', + '- Expected\n+ Received\n\n- 2\n+ 1', + "AssertionError: expected 'a' to be 'b' // Object.is equality", + 'Expected: "b"\nReceived: "a"', + ].join('\n\n'), + ); + expect(result?.diff).toBeUndefined(); + }); +}); diff --git a/packages/runtime/src/expect/diff.ts b/packages/runtime/src/expect/diff.ts new file mode 100644 index 00000000..64d830b0 --- /dev/null +++ b/packages/runtime/src/expect/diff.ts @@ -0,0 +1,24 @@ +import { processError } from '@vitest/utils/error'; + +const readDiff = (error: { diff?: unknown }): string | undefined => + typeof error.diff === 'string' && error.diff ? error.diff : undefined; + +// Vitest computes the expected/received diff in its test runner, so +// @vitest/expect only does it for soft assertions. Thrown assertion errors +// get the same treatment here. +export const getErrorDiff = (error: unknown): string | undefined => { + if (!error || typeof error !== 'object') { + return undefined; + } + + // Already processed, e.g. a soft assertion error. + if ('diff' in error) { + return readDiff(error); + } + + try { + return readDiff(processError(error)); + } catch { + return undefined; + } +}; diff --git a/packages/runtime/src/expect/errors.ts b/packages/runtime/src/expect/errors.ts index 7520ad72..03e1ce14 100644 --- a/packages/runtime/src/expect/errors.ts +++ b/packages/runtime/src/expect/errors.ts @@ -1,4 +1,5 @@ import type { HarnessExpectTestState } from './context.js'; +import { getErrorDiff } from './diff.js'; type SerializedExpectError = { name?: string; @@ -6,7 +7,7 @@ type SerializedExpectError = { stack?: string; }; -const formatErrorMessage = (error: unknown): string => { +const formatErrorTitle = (error: unknown): string => { if (error instanceof Error) { return `${error.name}: ${error.message}`; } @@ -21,6 +22,15 @@ const formatErrorMessage = (error: unknown): string => { return String(error); }; +// The flattened error has no actual/expected of its own, so each failure +// keeps its diff next to its message. +const formatErrorMessage = (error: unknown): string => { + const title = formatErrorTitle(error); + const diff = getErrorDiff(error); + + return diff ? `${title}\n\n${diff}` : title; +}; + const createExpectError = (errors: unknown[], title?: string): Error => { const message = [title, ...errors.map(formatErrorMessage)] .filter(Boolean) diff --git a/packages/runtime/src/runner/errors.ts b/packages/runtime/src/runner/errors.ts index 698366ab..bc40b65e 100644 --- a/packages/runtime/src/runner/errors.ts +++ b/packages/runtime/src/runner/errors.ts @@ -1,4 +1,5 @@ import type { SerializedError, CodeFrame } from '@react-native-harness/bridge'; +import { getErrorDiff } from '../expect/diff.js'; import { getCodeFrame } from '../symbolicate.js'; export class TestExecutionError extends Error { @@ -6,6 +7,7 @@ export class TestExecutionError extends Error { suite: string; test: string; codeFrame?: CodeFrame; + diff?: string; constructor( error: unknown, @@ -21,6 +23,7 @@ export class TestExecutionError extends Error { this.test = test; this.cause = error; this.codeFrame = codeFrame; + this.diff = getErrorDiff(error); } toSerializedJSON(): SerializedError { @@ -39,6 +42,7 @@ export class TestExecutionError extends Error { return { name: causeName, message: causeMessage, + diff: this.diff, codeFrame: causeCodeFrame, diagnostics: causeDiagnostics, }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3e41bc20..f36a302c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -600,6 +600,9 @@ importers: '@vitest/spy': specifier: 4.0.16 version: 4.0.16 + '@vitest/utils': + specifier: 4.0.16 + version: 4.0.16 chai: specifier: ^6.2.2 version: 6.2.2