feat: add run telemetry probes to diagnostics

.nx/version-plans/version-plan-1792133861187.md

@@ -0,0 +1,5 @@
+---
+__default__: minor
+---
+
+Run diagnostics now include resource telemetry. Set `diagnostics` to `true` or to an object (`enabled`, `probes`, `sampleIntervalMs`) to sample host load, CPU and memory pressure, Harness process memory and event loop lag, per-milestone process listings on macOS, and the process count and last boot stage of iOS simulators. Entries are written to `.harness/diagnostics/<sessionId>.jsonl` as the run progresses, so cancelled or timed-out CI jobs keep their data. Metrics appear as graphed tracks in the Perfetto trace, and a key-number summary (boot wait, time to ready, peak load, peak compressed memory, swapouts, event loop lag) is printed, saved to `<runId>.summary.json` and added to the GitHub Actions job summary.

packages/config/src/__tests__/diagnostics-config.test.ts

@@ -0,0 +1,128 @@
+import { describe, expect, it } from 'vitest';
+import {
+  ConfigSchema,
+  isDiagnosticsEnabled,
+  resolveDiagnosticsOptions,
+} from '../types.js';
+
+const baseConfig = {
+  entryPoint: './index.js',
+  appRegistryComponentName: 'App',
+  runners: [
+    {
+      name: 'test-runner',
+      config: {},
+      runner: 'test-runner',
+      platformId: 'test-platform',
+    },
+  ],
+};
+
+const parse = (diagnostics?: unknown) =>
+  ConfigSchema.parse({ ...baseConfig, diagnostics });
+
+describe('ConfigSchema diagnostics', () => {
+  it('defaults to false', () => {
+    expect(parse().diagnostics).toBe(false);
+  });
+
+  it('accepts the boolean form', () => {
+    expect(parse(true).diagnostics).toBe(true);
+    expect(parse(false).diagnostics).toBe(false);
+  });
+
+  it('accepts the object form', () => {
+    const diagnostics = {
+      enabled: true,
+      probes: { host: true, node: false, snapshots: false, platform: true },
+      sampleIntervalMs: 2000,
+    };
+
+    expect(parse(diagnostics).diagnostics).toEqual(diagnostics);
+    expect(parse({}).diagnostics).toEqual({});
+  });
+
+  it.each([
+    ['an unknown key', { enable: true }],
+    ['an unknown probe', { probes: { cpu: true } }],
+    ['a non-boolean probe switch', { probes: { host: 'yes' } }],
+    ['a too small interval', { sampleIntervalMs: 10 }],
+    ['a fractional interval', { sampleIntervalMs: 1000.5 }],
+    ['a string', 'true'],
+  ])('rejects %s', (_name, diagnostics) => {
+    expect(() => parse(diagnostics)).toThrow();
+  });
+});
+
+describe('isDiagnosticsEnabled', () => {
+  it('is off without config or environment', () => {
+    expect(isDiagnosticsEnabled(undefined, {})).toBe(false);
+    expect(isDiagnosticsEnabled({ diagnostics: false }, {})).toBe(false);
+  });
+
+  it('is on for true and for an object unless it opts out', () => {
+    expect(isDiagnosticsEnabled({ diagnostics: true }, {})).toBe(true);
+    expect(isDiagnosticsEnabled({ diagnostics: {} }, {})).toBe(true);
+    expect(
+      isDiagnosticsEnabled({ diagnostics: { sampleIntervalMs: 1000 } }, {})
+    ).toBe(true);
+    expect(isDiagnosticsEnabled({ diagnostics: { enabled: false } }, {})).toBe(
+      false
+    );
+  });
+
+  it.each(['1', 'true', 'yes'])(
+    'RN_HARNESS_DIAGNOSTICS=%s enables it',
+    (value) => {
+      expect(
+        isDiagnosticsEnabled(
+          { diagnostics: false },
+          { RN_HARNESS_DIAGNOSTICS: value }
+        )
+      ).toBe(true);
+    }
+  );
+
+  it.each(['', '0', 'false'])('RN_HARNESS_DIAGNOSTICS=%j does not', (value) => {
+    expect(
+      isDiagnosticsEnabled(
+        { diagnostics: false },
+        { RN_HARNESS_DIAGNOSTICS: value }
+      )
+    ).toBe(false);
+  });
+});
+
+describe('resolveDiagnosticsOptions', () => {
+  const allOn = { host: true, node: true, snapshots: true, platform: true };
+
+  it('turns every probe on by default for every form', () => {
+    for (const diagnostics of [false, true, {}]) {
+      expect(resolveDiagnosticsOptions({ diagnostics })).toEqual({
+        probes: allOn,
+        sampleIntervalMs: undefined,
+      });
+    }
+  });
+
+  it('resolves defaults without a config', () => {
+    expect(resolveDiagnosticsOptions(undefined)).toEqual({
+      probes: allOn,
+      sampleIntervalMs: undefined,
+    });
+  });
+
+  it('applies overrides and keeps the rest on', () => {
+    expect(
+      resolveDiagnosticsOptions({
+        diagnostics: {
+          probes: { snapshots: false },
+          sampleIntervalMs: 1000,
+        },
+      })
+    ).toEqual({
+      probes: { ...allOn, snapshots: false },
+      sampleIntervalMs: 1000,
+    });
+  });
+});

packages/config/src/index.ts

@@ -1,9 +1,10 @@
 export { getConfig } from './reader.js';
-export type { Config } from './types.js';
+export type { Config, ResolvedDiagnosticsOptions } from './types.js';
 export {
   ConfigSchema,
   DEFAULT_METRO_PORT,
   isDiagnosticsEnabled,
+  resolveDiagnosticsOptions,
   resolveMetroCacheEnabled,
   resolveSkipAlreadyIncludedModules,
 } from './types.js';

packages/config/src/types.ts

@@ -224,14 +224,66 @@ export const ConfigSchema = z
       ),
 
     diagnostics: z
-      .boolean()
+      .union([
+        z.boolean(),
+        z
+          .object({
+            enabled: z
+              .boolean()
+              .optional()
+              .describe(
+                'Turns diagnostics on or off. Defaults to true when the object form is used.'
+              ),
+            probes: z
+              .object({
+                host: z
+                  .boolean()
+                  .optional()
+                  .describe(
+                    'Sample host load, CPU split and memory pressure. Defaults to true.'
+                  ),
+                node: z
+                  .boolean()
+                  .optional()
+                  .describe(
+                    'Sample Harness process memory and event loop lag. Defaults to true.'
+                  ),
+                snapshots: z
+                  .boolean()
+                  .optional()
+                  .describe(
+                    'Record a process listing at run milestones (macOS). Defaults to true.'
+                  ),
+                platform: z
+                  .boolean()
+                  .optional()
+                  .describe(
+                    'Record platform specific measurements, such as the process count inside an iOS simulator. Defaults to true.'
+                  ),
+              })
+              .strict()
+              .optional(),
+            sampleIntervalMs: z
+              .number()
+              .int()
+              .min(250, 'sampleIntervalMs must be at least 250')
+              .optional()
+              .describe(
+                'How often host and process metrics are sampled, in milliseconds. Defaults to 5000.'
+              ),
+          })
+          .strict(),
+      ])
       .optional()
       .default(false)
       .describe(
-        'Enable diagnostics tracing for the harness session. Records spans for ' +
-          'session setup, Metro bundling, bridge/client events, and per-file test runs, ' +
-          'then writes a Chrome Trace Event JSON file and prints a summary after each run. ' +
-          'Can also be enabled via the RN_HARNESS_DIAGNOSTICS environment variable.'
+        'Enable diagnostics for the harness session. Records spans for session ' +
+          'setup, Metro bundling, bridge/client events, and per-file test runs, ' +
+          'samples host and process resource metrics, then writes a Chrome Trace ' +
+          'Event JSON file, an incremental JSONL log and a KPI summary, and prints ' +
+          'a summary after each run. Pass an object to tune the probes and the ' +
+          'sampling interval. Can also be enabled via the RN_HARNESS_DIAGNOSTICS ' +
+          'environment variable.'
       ),
 
     // Deprecated property - used for migration detection
@@ -256,21 +308,59 @@ export type Config = z.infer<typeof ConfigSchema>;
 
 /**
  * Resolves whether diagnostics tracing is enabled: either explicitly via the
- * `diagnostics` config option, or via the `RN_HARNESS_DIAGNOSTICS` environment
- * variable (any value other than unset, `''`, `'0'`, or `'false'`).
+ * `diagnostics` config option (`true`, or an object that does not set
+ * `enabled: false`), or via the `RN_HARNESS_DIAGNOSTICS` environment variable
+ * (any value other than unset, `''`, `'0'`, or `'false'`).
  */
 export const isDiagnosticsEnabled = (
   config: Pick<Config, 'diagnostics'> | undefined,
   env: NodeJS.ProcessEnv = process.env
 ): boolean => {
-  if (config?.diagnostics === true) {
+  const diagnostics = config?.diagnostics;
+  if (diagnostics === true) {
+    return true;
+  }
+  if (typeof diagnostics === 'object' && diagnostics.enabled !== false) {
     return true;
   }
 
   const envValue = env.RN_HARNESS_DIAGNOSTICS;
   return !!envValue && envValue !== '0' && envValue !== 'false';
 };
 
+export type ResolvedDiagnosticsOptions = {
+  probes: {
+    host: boolean;
+    node: boolean;
+    snapshots: boolean;
+    platform: boolean;
+  };
+  /** Undefined means "use the default sampling interval". */
+  sampleIntervalMs: number | undefined;
+};
+
+/**
+ * Resolves the diagnostics tuning options. Every probe is on unless the object
+ * form of `diagnostics` switches it off, which is also what the boolean form
+ * and the `RN_HARNESS_DIAGNOSTICS` environment variable mean.
+ */
+export const resolveDiagnosticsOptions = (
+  config: Pick<Config, 'diagnostics'> | undefined
+): ResolvedDiagnosticsOptions => {
+  const diagnostics = config?.diagnostics;
+  const options = typeof diagnostics === 'object' ? diagnostics : undefined;
+
+  return {
+    probes: {
+      host: options?.probes?.host ?? true,
+      node: options?.probes?.node ?? true,
+      snapshots: options?.probes?.snapshots ?? true,
+      platform: options?.probes?.platform ?? true,
+    },
+    sampleIntervalMs: options?.sampleIntervalMs,
+  };
+};
+
 /**
  * Resolves the effective value of `skipAlreadyIncludedModules`, reconciling
  * it with the deprecated `unstable__skipAlreadyIncludedModules` alias:

packages/jest/src/__tests__/execute-run.test.ts

@@ -12,7 +12,11 @@ import {
   AppBridgeDisconnectedError,
   DeviceNotRespondingError,
 } from '@react-native-harness/bridge/server';
-import { createDiagnostics } from '@react-native-harness/tools';
+import {
+  createDiagnostics,
+  getEmitter,
+  type LifecycleEvent,
+} from '@react-native-harness/tools';
 import type { HarnessSession } from '../harness-session.js';
 import { executeRun } from '../execute-run.js';
 
@@ -36,9 +40,15 @@ vi.mock('../run.js', () => ({
 // asserting on them doesn't race with executeRun's own diagnostics.flush().
 const mockPrintSummary = vi.hoisted(() => vi.fn());
 const mockWriteTraceFile = vi.hoisted(() => vi.fn(async () => '/fake/trace.json'));
-vi.mock('../diagnostics/index.js', () => ({
+const mockWriteSummaryFile = vi.hoisted(() => vi.fn(async () => '/fake/summary.json'));
+const mockAppendStepSummary = vi.hoisted(() => vi.fn(async () => undefined));
+vi.mock('../diagnostics/index.js', async (importOriginal) => ({
+  // summarizeRunKpis stays real: it is a pure reduction over the entries.
+  ...(await importOriginal<typeof import('../diagnostics/index.js')>()),
   printSummary: mockPrintSummary,
   writeTraceFile: mockWriteTraceFile,
+  writeSummaryFile: mockWriteSummaryFile,
+  appendStepSummary: mockAppendStepSummary,
 }));
 
 // ---------------------------------------------------------------------------
@@ -125,6 +135,8 @@ const makeSession = (overrides: Partial<HarnessSession> = {}): HarnessSession =>
     },
   } as HarnessSession['context'],
   diagnostics: createDiagnostics({ enabled: false }),
+  sessionId: 'session-1',
+  lifecycle: getEmitter<LifecycleEvent>(),
   ensureAppReady: vi.fn(resolveUndefined),
   runTestFile: vi.fn(async () => makeHarnessResult()),
   restartApp: vi.fn(resolveUndefined),
@@ -231,6 +243,22 @@ describe('executeRun', () => {
       expect(mockWriteTraceFile).toHaveBeenCalledWith(entries, expect.objectContaining({ runId: expect.any(String) }));
     });
 
+    it('writes the KPI summary and step summary next to the trace', async () => {
+      mockWriteSummaryFile.mockClear();
+      mockAppendStepSummary.mockClear();
+      const diagnostics = createDiagnostics({ enabled: true });
+      const session = makeSession({ diagnostics });
+
+      await executeRun(session, [makeTest('/a.ts')], makeWatcher(), makeEmitEvent().emitEvent, makeGlobalConfig());
+
+      const kpis = { testFileCount: 1, result: 'passed' };
+      expect(mockWriteSummaryFile).toHaveBeenCalledWith(
+        expect.objectContaining(kpis),
+        expect.objectContaining({ runId: expect.any(String), sessionId: 'session-1' }),
+      );
+      expect(mockAppendStepSummary).toHaveBeenCalledWith(expect.objectContaining(kpis));
+    });
+
     it('still records run.total when a test throws', async () => {
       const diagnostics = createDiagnostics({ enabled: true });
       const session = makeSession({
@@ -247,6 +275,24 @@ describe('executeRun', () => {
       });
     });
 
+    it('emits run and test-file lifecycle phases', async () => {
+      const session = makeSession();
+      const phases: Array<[string, unknown]> = [];
+      session.lifecycle.addListener((event) =>
+        phases.push([event.phase, event.attrs]),
+      );
+
+      await executeRun(session, [makeTest('/a.ts')], makeWatcher(), makeEmitEvent().emitEvent, makeGlobalConfig());
+
+      const file = path.join('..', 'a.ts');
+      expect(phases).toEqual([
+        ['run.start', { runId: expect.any(String) }],
+        ['test-file.start', { file }],
+        ['test-file.end', { file, status: 'passed' }],
+        ['run.end', { runId: expect.any(String), status: 'passed' }],
+      ]);
+    });
+
     it('is a no-op and does not generate a report when diagnostics is disabled', async () => {
       const diagnostics = createDiagnostics({ enabled: false });
       const session = makeSession({ diagnostics });

packages/jest/src/__tests__/index.test.ts

@@ -3,7 +3,12 @@ import type { Config, Test, TestEvents, TestRunnerOptions, TestWatcher } from 'j
 import type { TestSuiteResult } from '@react-native-harness/bridge';
 import type { HarnessSession } from '../harness-session.js';
 import type { executeRun as ExecuteRun } from '../execute-run.js';
-import { createDiagnostics, HarnessError } from '@react-native-harness/tools';
+import {
+  createDiagnostics,
+  getEmitter,
+  HarnessError,
+  type LifecycleEvent,
+} from '@react-native-harness/tools';
 import JestHarness from '../index.js';
 
 const resolveUndefined = async () => undefined;
@@ -16,6 +21,8 @@ const mockSession: HarnessSession = {
   config: { metroPort: 8081 } as HarnessSession['config'],
   context: {} as HarnessSession['context'],
   diagnostics: createDiagnostics({ enabled: false }),
+  sessionId: 'session-1',
+  lifecycle: getEmitter<LifecycleEvent>(),
   onTestRunnerEvent: vi.fn(() => () => undefined),
   ensureAppReady: vi.fn(resolveUndefined),
   runTestFile: vi.fn(async (): Promise<TestSuiteResult> => ({