From 4837698d1dfabb0dbce109be4326b7e19f355067 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Wed, 4 Nov 2026 16:53:59 -0800 Subject: [PATCH 1/2] test(frontend): close technique selection acceptance gap Exercise real preset editing, scenario defaults, estimate recovery and launch with isolated provider-free data. Retain desktop/narrow provenance and execution evidence, explain disabled choices, and use existing touch targets for preview actions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/frontend_tests.yml | 6 +- frontend/README.md | 44 ++ .../fixtures/scenario_techniques_backend.py | 279 ++++++++++++ frontend/e2e/scenario-techniques.spec.ts | 419 ++++++++++++++++++ frontend/package.json | 1 + frontend/playwright.config.ts | 2 +- .../playwright.scenario-techniques.config.ts | 32 ++ .../ScenarioPresetEditor.test.tsx | 11 + .../Scenarios/ScenarioDetail.test.tsx | 38 +- .../components/Scenarios/ScenarioDetail.tsx | 27 +- .../Scenarios/ScenarioTechniqueSelector.tsx | 4 +- .../test_scenario_techniques_fixture.py | 52 +++ 12 files changed, 905 insertions(+), 10 deletions(-) create mode 100644 frontend/e2e/fixtures/scenario_techniques_backend.py create mode 100644 frontend/e2e/scenario-techniques.spec.ts create mode 100644 frontend/playwright.scenario-techniques.config.ts create mode 100644 tests/unit/backend/test_scenario_techniques_fixture.py diff --git a/.github/workflows/frontend_tests.yml b/.github/workflows/frontend_tests.yml index 4f572016aa..fac6923e59 100644 --- a/.github/workflows/frontend_tests.yml +++ b/.github/workflows/frontend_tests.yml @@ -86,6 +86,10 @@ jobs: config: playwright.scenario-hierarchy.config.ts shard: 1 total: 1 + - project: scenario-techniques + config: playwright.scenario-techniques.config.ts + shard: 1 + total: 1 outputs: expected-reports: ${{ strategy.job-total }} permissions: @@ -131,7 +135,7 @@ jobs: run: npx playwright install --with-deps chromium - name: Build the isolated fixture frontend - if: matrix.project == 'analytics' || matrix.project == 'scenario-hierarchy' + if: matrix.project == 'analytics' || matrix.project == 'scenario-hierarchy' || matrix.project == 'scenario-techniques' run: npm run build - name: Run E2E shard (no credentials needed) diff --git a/frontend/README.md b/frontend/README.md index 2dcd6de8e0..86da16cf30 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -244,6 +244,50 @@ piece IDs and 121 references. Journeys cover bounded requests, continuation retry/restart failures, pending-reader ownership, URL/History context and keyboard/touch use. `PYRIT_E2E_HIERARCHY_PORT` overrides the loopback port. +### Isolated Scenario technique browser checks + +From `frontend`, run: + +```powershell +$env:PYRIT_PYTHON = uv run --no-sync python -c "import sys; print(sys.executable)" +npm run test:e2e:scenario-techniques +``` + +This builds the current frontend and exercises real preset storage, scenario +metadata, estimates, launch, attacks, scoring and saved run plans over disposable +SQLite data. The fixture uses a private home, empty environment-file and +initializer lists, inline objectives, local encoding converters, a deterministic +target and `SubStringScorer`. It does not call providers or use a developer's +configuration. The fixture explicitly registers its three local scenarios instead +of discovering provider-backed built-ins, and enables local administrator access +only in that isolated process so preset CRUD follows the real authorization path. +Configuration paths are pinned before application startup. Preset directories, +documents and lock sidecars are checked for containment before mutation; each +journey also checks the backend's database/configuration/preset-source paths. +`PYRIT_E2E_TECHNIQUES_PORT` overrides its loopback port (18611). +The containment regressions can also be run from the repository root: + +```powershell +uv run --no-sync pytest tests\unit\backend\test_scenario_techniques_fixture.py -q +``` + +Desktop and narrow/touch journeys cover preset selection and editing, +scenario-relative defaults, aggregate expansion, independent baseline settings, +keyboard/focus recovery, 44px preview actions for coarse-pointer input, and +completed execution. Touch measurements wait for the dialog to settle, without +fixed sleeps. Only loading and error checks +inject faults: the fixture holds one estimate or returns one deliberate 503, +then delegates successful estimates to the real service. No browser API +responses are intercepted. CI runs this as a separate isolated E2E shard. + +Screenshots and their source-head, compatibility, viewport, state and fault +provenance, plus the actual launch request, stored configuration and execution +plan/results, are attached to the Playwright report. Set +`PYRIT_TECHNIQUES_SCREENSHOT_DIR` to retain the PNG/JSON pairs separately. +For exact-head evidence, commit first, then rebuild and rerun from that clean +head. The provenance records any remaining worktree changes rather than +attributing uncommitted code to a commit. + ## Development ```bash diff --git a/frontend/e2e/fixtures/scenario_techniques_backend.py b/frontend/e2e/fixtures/scenario_techniques_backend.py new file mode 100644 index 0000000000..058db4c3dc --- /dev/null +++ b/frontend/e2e/fixtures/scenario_techniques_backend.py @@ -0,0 +1,279 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Real preset, estimate, and execution routes over disposable, provider-free data.""" + +from __future__ import annotations + +import asyncio +import os +from contextlib import ExitStack, contextmanager +from pathlib import Path +from typing import TYPE_CHECKING, Literal +from unittest.mock import patch + +from pydantic import BaseModel + +from frontend.e2e.fixtures import analytics_backend + +if TYPE_CHECKING: + from collections.abc import Generator + + from pyrit.memory import SQLiteMemory + from pyrit.models.catalog.scenario import ScenarioRunSizeEstimate, ScenarioRunSizeEstimateRequest + from pyrit.scenario.core.scenario_context import ScenarioContext + + +class EstimateControl(BaseModel): + """Explicit one-shot faults for the otherwise real estimate service.""" + + action: Literal["reset", "status", "hold", "release", "fail"] + + +def _register_components() -> None: + from pyrit.converter import Base64Converter, ROT13Converter + from pyrit.executor.attack import AttackConverterConfig, PromptSendingAttack + from pyrit.models import Message, MessagePiece, SeedObjective + from pyrit.prompt_normalizer import ConverterConfiguration + from pyrit.prompt_target import PromptTarget, TargetCapabilities, TargetConfiguration + from pyrit.registry import AttackTechniqueRegistry, ScenarioRegistry, TargetRegistry + from pyrit.scenario.core import ( + AtomicAttack, + AttackTechniqueFactory, + BaselineAttackPolicy, + DatasetAttackConfiguration, + Scenario, + ) + from pyrit.scenario.core.matrix_atomic_attack_builder import build_matrix_atomic_attacks + from pyrit.score import SubStringScorer + + class TechniqueAcceptanceTarget(PromptTarget): + """Respond locally so real attacks and scores can be persisted.""" + + _DEFAULT_CONFIGURATION = TargetConfiguration( + capabilities=TargetCapabilities( + supports_multi_turn=True, + supports_multi_message_pieces=True, + supports_system_prompt=True, + supports_editable_history=True, + ) + ) + + async def _send_prompt_to_target_async(self, *, normalized_conversation: list[Message]) -> list[Message]: + return [ + MessagePiece( + role="assistant", + original_value="Offline technique acceptance response.", + conversation_id=normalized_conversation[-1].message_pieces[0].conversation_id, + ).to_message() + ] + + def _validate_request(self, *, normalized_conversation: list[Message]) -> None: + if not normalized_conversation: + raise ValueError("The offline acceptance target requires a message.") + + factories = [ + AttackTechniqueFactory( + name="offline_plain", + attack_class=PromptSendingAttack, + description="Send the local objective unchanged.", + technique_tags=["local", "single_turn"], + supports_additional_request_converters=True, + ), + *[ + AttackTechniqueFactory( + name=name, + attack_class=PromptSendingAttack, + description=f"Apply the local {converter.__class__.__name__} before sending.", + technique_tags=["local", "encoded", "single_turn"], + supports_additional_request_converters=True, + attack_kwargs={ + "attack_converter_config": AttackConverterConfig( + request_converters=ConverterConfiguration.from_converters(converters=[converter]) + ), + }, + ) + for name, converter in [("offline_base64", Base64Converter()), ("offline_rot13", ROT13Converter())] + ], + ] + techniques = AttackTechniqueRegistry.get_registry_singleton() + techniques.register_from_factories(factories) + + class TechniqueAcceptanceScenario(Scenario): + """A local technique sweep with a scenario-relative default and optional baseline.""" + + VERSION = 1 + MAX_CONCRETE_TECHNIQUES = 2 + TECHNIQUE_CLASS = techniques.build_technique_class_from_factories( + class_name="TechniqueAcceptanceScenarioTechnique", + factories=factories, + default_names={"offline_plain"}, + ) + + def __init__(self, *, scenario_result_id: str | None = None) -> None: + super().__init__( + version=self.VERSION, + technique_class=self.TECHNIQUE_CLASS, + default_dataset_config=DatasetAttackConfiguration( + seeds=[SeedObjective(value="Reply to this harmless local acceptance objective.")], + max_total=1, + ), + objective_scorer=SubStringScorer(substring="Offline technique acceptance"), + scenario_result_id=scenario_result_id, + uses_default_adversarial_target=False, + ) + + async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list[AtomicAttack]: + return await asyncio.to_thread( + build_matrix_atomic_attacks, context=context, objective_scorer=self._objective_scorer + ) + + class TechniqueNoBaselineScenario(TechniqueAcceptanceScenario): + """The same pool, with a different default and no baseline support.""" + + BASELINE_ATTACK_POLICY = BaselineAttackPolicy.Forbidden + TECHNIQUE_CLASS = techniques.build_technique_class_from_factories( + class_name="TechniqueNoBaselineScenarioTechnique", + factories=factories, + default_names={"offline_rot13"}, + ) + + class TechniqueOptionalBaselineScenario(TechniqueAcceptanceScenario): + """The same pool, with a third default and baseline initially off.""" + + BASELINE_ATTACK_POLICY = BaselineAttackPolicy.Disabled + TECHNIQUE_CLASS = techniques.build_technique_class_from_factories( + class_name="TechniqueOptionalBaselineScenarioTechnique", + factories=factories, + default_names={"offline_base64"}, + ) + + targets = TargetRegistry.get_registry_singleton().instances + for name in ["technique_offline", "technique_offline_alternate"]: + targets.register(TechniqueAcceptanceTarget(), name=name) + scenarios = ScenarioRegistry.get_registry_singleton() + for name, scenario_class in [ + ("technique_acceptance", TechniqueAcceptanceScenario), + ("technique_no_baseline", TechniqueNoBaselineScenario), + ("technique_optional_baseline", TechniqueOptionalBaselineScenario), + ]: + scenarios.register_class(scenario_class, name=name) + + +def _guard_preset_paths(*, patches: ExitStack, directory: Path) -> None: + from pyrit.registry.scenario_preset_storage import ScenarioPresetStorage + + root = directory.resolve() + original_directory = ScenarioPresetStorage._local_directory + original_lock = ScenarioPresetStorage._local_document_lock + + def local_directory(storage: ScenarioPresetStorage, *, create: bool = False) -> Path: + resolved = original_directory(storage, create=False).resolve() + if not resolved.is_relative_to(root): + raise RuntimeError(f"Refusing preset storage outside the fixture: {resolved}") + return original_directory(storage, create=create) + + @contextmanager + def document_lock(storage: ScenarioPresetStorage, path: Path) -> Generator[None, None, None]: + for location in (path, path.with_name(f".{path.name}{storage.LOCK_SUFFIX}")): + if not location.resolve().is_relative_to(root): + raise RuntimeError(f"Refusing preset mutation outside the fixture: {location}") + with original_lock(storage, path): + yield + + patches.enter_context(patch.object(ScenarioPresetStorage, "_local_directory", local_directory)) + patches.enter_context(patch.object(ScenarioPresetStorage, "_local_document_lock", document_lock)) + + +def _install_estimate_controls(*, patches: ExitStack, memory: SQLiteMemory) -> None: + from fastapi import HTTPException + + from pyrit.backend.main import app + from pyrit.backend.services.scenario_service import get_scenario_service + from pyrit.common.path import CONFIGURATION_DIRECTORY_PATH + from pyrit.registry import ScenarioPresetRegistry + + isolation = { + "fixture_root": str(Path(memory.db_path).resolve().parents[1]), + "database_path": str(Path(memory.db_path).resolve()), + "configuration_root": str(CONFIGURATION_DIRECTORY_PATH.resolve()), + "preset_source": ScenarioPresetRegistry.get_registry_singleton().source, + } + service = get_scenario_service() + original_estimate = service.estimate_scenario_run_size_async + hold_next = False + fail_next = False + waiting = False + release = asyncio.Event() + release.set() + + async def control_async(command: EstimateControl) -> dict[str, bool | dict[str, str]]: + nonlocal hold_next, fail_next + if command.action == "reset": + hold_next = fail_next = False + release.set() + elif command.action == "hold": + hold_next = True + release.clear() + elif command.action == "release": + release.set() + elif command.action == "fail": + fail_next = True + return {"waiting": waiting, "isolation": isolation} + + async def estimate_async( + *, scenario_name: str, request: ScenarioRunSizeEstimateRequest + ) -> ScenarioRunSizeEstimate | None: + nonlocal hold_next, fail_next, waiting + if scenario_name.startswith("technique_"): + if hold_next: + hold_next = False + waiting = True + try: + await release.wait() + finally: + waiting = False + if fail_next: + fail_next = False + raise HTTPException( + status_code=503, detail="Deliberate offline estimate failure. Try another estimate." + ) + return await original_estimate(scenario_name=scenario_name, request=request) + + patches.enter_context(patch.object(service, "estimate_scenario_run_size_async", estimate_async)) + app.add_api_route("/api/e2e/techniques/estimate-control", control_async, methods=["POST"]) + + +if __name__ == "__main__": + with ExitStack() as patches: + original_serve = analytics_backend._serve + + def _serve(*, directory: Path, port: int) -> None: + from pyrit.common import path as paths + from pyrit.registry import ScenarioRegistry + + # Only the three explicitly registered local scenarios belong to this deployment. + with ( + patch.dict(os.environ, {"PYRIT_ALLOW_UNAUTHENTICATED_ADMIN": "true"}), + patch.object(paths, "CONFIGURATION_DIRECTORY_PATH", directory / ".pyrit"), + patch.object(paths, "DEFAULT_CONFIG_PATH", directory / ".pyrit" / paths.DEFAULT_CONFIG_FILENAME), + patch.object(ScenarioRegistry, "_discover", return_value=None), + ): + _guard_preset_paths(patches=patches, directory=directory) + original_serve(directory=directory, port=port) + + async def _seed_async(memory: SQLiteMemory) -> None: + from pyrit.registry import ScenarioPresetRegistry + + memory.disable_embedding() + await asyncio.to_thread(_register_components) + # Preset writes, like the database, must stay inside the disposable fixture. + presets = ScenarioPresetRegistry.get_registry_singleton() + presets.configure_source(str(Path(memory.db_path).parent / "scenario_presets")) + if not Path(presets.source).resolve().is_relative_to(Path(memory.db_path).resolve().parents[1]): + raise RuntimeError("The preset source must be inside the fixture before accepting requests.") + _install_estimate_controls(patches=patches, memory=memory) + + patches.enter_context(patch.object(analytics_backend, "_seed_async", _seed_async)) + patches.enter_context(patch.object(analytics_backend, "_serve", _serve)) + analytics_backend.main() diff --git a/frontend/e2e/scenario-techniques.spec.ts b/frontend/e2e/scenario-techniques.spec.ts new file mode 100644 index 0000000000..e9eef2d818 --- /dev/null +++ b/frontend/e2e/scenario-techniques.spec.ts @@ -0,0 +1,419 @@ +import { execFileSync } from 'node:child_process' +import { mkdir, writeFile } from 'node:fs/promises' +import path from 'node:path' + +import { expect, test, type Locator, type Page, type Response, type TestInfo } from '@playwright/test' + +import type { + RegisteredScenario, + RunScenarioRequest, + ScenarioPreset, + ScenarioPresetResponse, + ScenarioProgressResult, + ScenarioRunPlanAtomicGroup, + ScenarioRunProgress, + ScenarioRunSizeEstimateRequest, + ScenarioRunSizeEstimateResponse, + ScenarioRunSummary, +} from '../src/types' +import { presetEditorRoutePath, presetScannerRoutePath, PRESETS_ROUTE } from '../src/components/ScenarioPresets/presetRoutes' +import { scenarioRunRoutePath } from '../src/utils/routeParams' +import { compatibilityHeaders, getCompatibilityId } from './_compatibility' + +const SCENARIO = 'technique_acceptance' +const TECHNIQUES = ['offline_plain', 'offline_base64', 'offline_rot13'] +const TARGET = 'technique_offline' +const ALTERNATE_TARGET = 'technique_offline_alternate' +const FAILURE = 'Deliberate offline estimate failure. Try another estimate.' +const VIEWPORTS = [ + { name: 'desktop', viewport: { width: 1440, height: 1050 }, touch: false }, + { name: 'narrow', viewport: { width: 390, height: 844 }, touch: true }, +] + +type EstimateAction = 'reset' | 'status' | 'hold' | 'release' | 'fail' +interface EstimateControlState { + waiting: boolean + isolation: { + fixture_root: string + database_path: string + configuration_root: string + preset_source: string + } +} + +async function control(page: Page, action: EstimateAction): Promise { + const response = await page.request.post('/api/e2e/techniques/estimate-control', { + headers: compatibilityHeaders(), data: { action }, + }) + expect(response.status(), await response.text()).toBe(200) + return response.json() +} + +function nextEstimate(page: Page, scenario: string = SCENARIO): Promise { + return page.waitForResponse((response: Response): boolean => + new URL(response.url()).pathname === `/api/scenarios/catalog/${scenario}/estimate` + && response.request().method() === 'POST', + ) +} + +async function readEstimate(response: Response, count: number, baseline: boolean): Promise { + expect(response.status(), await response.text()).toBe(200) + const estimate: ScenarioRunSizeEstimateResponse = await response.json() + expect(estimate.status).toBe('approximate') + expect(estimate.estimated_attack_count).toBe(count) + expect(estimate.components.some( + (component: ScenarioRunSizeEstimateResponse['components'][number]): boolean => component.is_baseline, + )).toBe(baseline) + return estimate +} + +async function createPreset(page: Page, preset: ScenarioPreset): Promise { + const response = await page.request.post('/api/scenario-presets', { headers: compatibilityHeaders(), data: preset }) + expect(response.status(), await response.text()).toBe(201) + const saved: ScenarioPresetResponse = await response.json() + expect(saved.issues).toEqual([]) + expect(saved.origin).toBe('stored') +} + +async function readPreset(page: Page, name: string): Promise { + const response = await page.request.get(`/api/scenario-presets/${name}`, { headers: compatibilityHeaders() }) + expect(response.status(), await response.text()).toBe(200) + return response.json() +} + +async function expectSelection(page: Page, techniques: string[], baseline: boolean): Promise { + const group = page.getByRole('group', { name: 'Techniques', exact: true }) + for (const name of TECHNIQUES) { + if (techniques.includes(name)) await expect(group.getByRole('checkbox', { name, exact: true })).toBeChecked() + else await expect(group.getByRole('checkbox', { name, exact: true })).not.toBeChecked() + } + if (baseline) await expect(group.getByRole('checkbox', { name: 'baseline', exact: true })).toBeChecked() + else await expect(group.getByRole('checkbox', { name: 'baseline', exact: true })).not.toBeChecked() +} + +async function tabTo(page: Page, target: Locator): Promise { + for (let index = 0; index < 80; index += 1) { + await page.keyboard.press('Tab') + if (await target.evaluate((element: Element): boolean => element === document.activeElement)) return + } + await expect(target, 'The control must be reachable with Tab.').toBeFocused() +} + +async function expectUnclipped(page: Page, target: Locator): Promise { + await target.evaluate((element: Element): void => { + element.scrollIntoView({ block: 'center', inline: 'nearest', behavior: 'instant' }) + }) + await expect(target).toBeVisible() + const bounds = await target.boundingBox() + expect(bounds).not.toBeNull() + if (!bounds) throw new Error('The visible control has no browser bounds.') + const viewport = page.viewportSize() + if (!viewport) throw new Error('The test requires a fixed viewport.') + expect(bounds.x).toBeGreaterThanOrEqual(0) + expect(bounds.x + bounds.width).toBeLessThanOrEqual(viewport.width) + expect(bounds.y).toBeGreaterThanOrEqual(0) + expect(bounds.y + bounds.height).toBeLessThanOrEqual(viewport.height) +} + +async function capture(page: Page, testInfo: TestInfo, viewName: string, state: string): Promise { + const name = `${viewName}-${state}` + const directory = process.env.PYRIT_TECHNIQUES_SCREENSHOT_DIR ?? testInfo.outputDir + await mkdir(directory, { recursive: true }) + const screenshot = path.join(directory, `${name}.png`) + const head = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim() + const status = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf8' }).trim() + expect(getCompatibilityId()).toMatch(new RegExp(`\\+g${head}$`)) + const provenance = { + state, test: testInfo.titlePath, retry: testInfo.retry, viewport: page.viewportSize(), url: page.url(), + captured_at: new Date().toISOString(), source_head: head, worktree_status: status, + compatibility_id: getCompatibilityId(), + fixture: 'scenario_techniques_backend.py; private SQLite and preset storage; local target and SubStringScorer', + isolation: (await control(page, 'status')).isolation, + fault_injection: state.includes('loading') ? 'One configured estimate deliberately held before the real service.' + : state.includes('error') ? 'One configured estimate deliberately returned HTTP 503.' : null, + } + await page.screenshot({ path: screenshot, animations: 'disabled' }) + await writeFile(path.join(directory, `${name}.json`), JSON.stringify(provenance, null, 2)) + await testInfo.attach(state, { path: screenshot, contentType: 'image/png' }) + await testInfo.attach(`${state}-provenance`, { body: JSON.stringify(provenance, null, 2), contentType: 'application/json' }) +} + +for (const view of VIEWPORTS) { + test.describe(view.name, (): void => { + test.use({ viewport: view.viewport, hasTouch: view.touch, isMobile: view.touch }) + + test.beforeEach(async ({ page }: { page: Page }): Promise => { + const { isolation } = await control(page, 'reset') + expect(path.relative(path.resolve('..'), isolation.fixture_root)).toMatch(/^\.pyrit_cache[\\/]/) + for (const location of [isolation.database_path, isolation.configuration_root, isolation.preset_source]) { + const relative = path.relative(isolation.fixture_root, location) + expect(path.isAbsolute(relative)).toBe(false) + expect(relative.split(path.sep)[0]).not.toBe('..') + } + }) + + test.afterEach(async ({ page }: { page: Page }): Promise => { + await control(page, 'reset') + }) + + test('edits a saved preset and preserves its selection through real estimates and execution', async ( + { page }: { page: Page }, testInfo: TestInfo, + ): Promise => { + const name = `techniques_${view.name}_${testInfo.retry}` + const errors: string[] = [] + page.on('pageerror', (error: Error): void => { errors.push(error.message) }) + await createPreset(page, { + name, scenario_name: SCENARIO, techniques: ['offline_plain'], include_baseline: true, + description: 'Original local acceptance preset.', + }) + + await page.goto(PRESETS_ROUTE) + await page.getByRole('region', { name: 'Default Labels' }).getByRole('button', { name: 'Clear', exact: true }).click() + await page.getByRole('link', { name, exact: true }).click() + await expect(page).toHaveURL(presetEditorRoutePath(name)) + await expect(page.getByTestId('scenario-preset-editor')).toBeVisible({ timeout: 30_000 }) + await expectSelection(page, ['offline_plain'], true) + await page.getByRole('checkbox', { name: 'offline_plain', exact: true }).uncheck() + await page.getByRole('checkbox', { name: 'offline_base64', exact: true }).check() + await page.getByRole('checkbox', { name: 'baseline', exact: true }).uncheck() + await page.getByRole('textbox', { name: 'Description', exact: true }).fill('Edited local technique selection.') + await page.getByRole('group', { name: 'Techniques', exact: true }).scrollIntoViewIfNeeded() + await capture(page, testInfo, view.name, 'preset-edited') + await page.getByRole('button', { name: 'Save preset', exact: true }).click() + await expect(page).toHaveURL(PRESETS_ROUTE) + const saved = await readPreset(page, name) + expect(saved.preset).toMatchObject({ + techniques: ['offline_base64'], include_baseline: false, description: 'Edited local technique selection.', + }) + + await page.goto('/scanner/launch') + await page.getByRole('link', { name, exact: true }).click() + await expect(page).toHaveURL(presetScannerRoutePath(SCENARIO, name)) + await expect(page.getByTestId('preset-seed-banner')).toContainText(name) + await expectSelection(page, ['offline_base64'], false) + const targeted = nextEstimate(page) + await page.getByRole('combobox', { name: 'Objective Target', exact: true }).selectOption(TARGET) + const targetedResponse = await targeted + expect(targetedResponse.request().postDataJSON()).toMatchObject({ + techniques: ['offline_base64'], include_baseline: false, target_name: TARGET, + }) + await readEstimate(targetedResponse, 1, false) + await tabTo(page, page.getByRole('checkbox', { name: 'offline_rot13', exact: true })) + const selected = nextEstimate(page) + await page.keyboard.press('Space') + const selectedResponse = await selected + expect(selectedResponse.request().postDataJSON()).toMatchObject({ + techniques: ['offline_base64', 'offline_rot13'], include_baseline: false, + }) + await readEstimate(selectedResponse, 2, false) + await expectSelection(page, ['offline_base64', 'offline_rot13'], false) + const plain = page.getByRole('checkbox', { name: 'offline_plain', exact: true }) + await expect(plain).toBeDisabled() + await expect(plain).toHaveAccessibleDescription(/at most 2.*[Cc]lear a selected technique/) + + await control(page, 'hold') + const held = nextEstimate(page) + await page.getByRole('combobox', { name: 'Objective Target', exact: true }).selectOption(ALTERNATE_TARGET) + await expect.poll(async (): Promise => (await control(page, 'status')).waiting).toBe(true) + await expectSelection(page, ['offline_base64', 'offline_rot13'], false) + const estimate = page.getByTestId('run-estimate') + await expect(estimate.getByRole('status')).toContainText('Updating estimate') + await estimate.scrollIntoViewIfNeeded() + await capture(page, testInfo, view.name, 'estimate-loading') + await control(page, 'release') + const heldResponse = await held + expect(heldResponse.request().postDataJSON()).toMatchObject({ + techniques: ['offline_base64', 'offline_rot13'], include_baseline: false, target_name: ALTERNATE_TARGET, + }) + await readEstimate(heldResponse, 2, false) + await expectSelection(page, ['offline_base64', 'offline_rot13'], false) + + await control(page, 'fail') + const failed = nextEstimate(page) + await page.getByRole('checkbox', { name: 'baseline', exact: true }).check() + const failedResponse = await failed + expect(failedResponse.status()).toBe(503) + expect(failedResponse.request().postDataJSON()).toMatchObject({ + techniques: ['offline_base64', 'offline_rot13'], include_baseline: true, target_name: ALTERNATE_TARGET, + }) + await expect(estimate.getByRole('status')).toContainText(FAILURE) + await expectSelection(page, ['offline_base64', 'offline_rot13'], true) + await estimate.scrollIntoViewIfNeeded() + await capture(page, testInfo, view.name, 'estimate-error') + + const refreshed = nextEstimate(page) + await page.getByRole('combobox', { name: 'Objective Target', exact: true }).selectOption(TARGET) + const refreshedResponse = await refreshed + await readEstimate(refreshedResponse, 3, true) + const estimateRequest: ScenarioRunSizeEstimateRequest = refreshedResponse.request().postDataJSON() + expect(estimateRequest).toMatchObject({ + techniques: ['offline_base64', 'offline_rot13'], include_baseline: true, target_name: TARGET, + }) + await expectSelection(page, ['offline_base64', 'offline_rot13'], true) + const baseline = page.getByRole('checkbox', { name: 'baseline', exact: true }) + await tabTo(page, baseline) + await expectUnclipped(page, baseline) + await expect(baseline).toBeFocused() + expect(await baseline.evaluate((element: Element): boolean => element.matches(':focus-visible'))).toBe(true) + await capture(page, testInfo, view.name, 'selection-keyboard-focus') + + const launch = page.getByRole('button', { name: 'Launch scan', exact: true }) + await expect(launch).toBeEnabled() + await tabTo(page, launch) + await expectUnclipped(page, launch) + await page.keyboard.press('Enter') + const preview = page.getByRole('dialog', { name: 'Run preview' }) + await expect(preview).toHaveCSS('opacity', '1') + await expect(preview.getByText('offline_base64', { exact: true })).toBeVisible() + await expect(preview.getByText('offline_rot13', { exact: true })).toBeVisible() + await expect(preview.getByText('baseline', { exact: true })).toBeVisible() + await expect(preview.getByText('offline_plain', { exact: true })).toHaveCount(0) + await page.keyboard.press('Escape') + await expect(preview).toHaveCount(0) + await expect(launch).toBeFocused() + await page.keyboard.press('Enter') + await expect(preview).toHaveCSS('opacity', '1') + const confirm = preview.getByRole('button', { name: 'Launch scan', exact: true }) + await tabTo(page, confirm) + await expectUnclipped(page, confirm) + if (view.touch) { + expect(await page.evaluate((): boolean => matchMedia('(pointer: coarse)').matches)).toBe(true) + for (const action of [confirm, preview.getByRole('button', { name: 'Cancel', exact: true })]) { + await expect.poll(async (): Promise => (await action.boundingBox())?.height) + .toBeGreaterThanOrEqual(44) + await expectUnclipped(page, action) + } + } + await expect(confirm).toBeFocused() + await capture(page, testInfo, view.name, 'run-preview') + const launched = page.waitForResponse((response: Response): boolean => + new URL(response.url()).pathname === '/api/scenarios/runs' && response.request().method() === 'POST', + ) + await page.keyboard.press('Enter') + const launchResponse = await launched + expect(launchResponse.status(), await launchResponse.text()).toBe(202) + const request: RunScenarioRequest = launchResponse.request().postDataJSON() + expect(request).toMatchObject({ ...estimateRequest, scenario_name: SCENARIO }) + const summary: ScenarioRunSummary = await launchResponse.json() + const runId = summary.scenario_result_id + await expect(page).toHaveURL(scenarioRunRoutePath(runId)) + await expect(page.getByTestId('run-state-badge')).toHaveText('Completed', { timeout: 30_000 }) + const configuration = await page.request.get(`/api/scenarios/runs/${runId}/launch-configuration`, { + headers: compatibilityHeaders(), + }) + expect(configuration.status()).toBe(200) + const savedConfiguration: RunScenarioRequest = await configuration.json() + expect(savedConfiguration).toMatchObject({ ...request }) + const progressResponse = await page.request.get(`/api/scenarios/runs/${runId}/progress`, { + headers: compatibilityHeaders(), + }) + expect(progressResponse.status()).toBe(200) + const progress: ScenarioRunProgress = await progressResponse.json() + expect(progress.plan_complete).toBe(true) + expect(progress.plan?.atomic_groups.filter( + (group: ScenarioRunPlanAtomicGroup): boolean => group.kind === 'baseline', + )).toHaveLength(1) + expect(progress.plan?.atomic_groups.filter((group: ScenarioRunPlanAtomicGroup): boolean => group.kind !== 'baseline') + .map((group: ScenarioRunPlanAtomicGroup): string | null | undefined => group.technique_name).sort()) + .toEqual(['offline_base64', 'offline_rot13']) + expect(progress.results).toHaveLength(3) + expect(progress.results.every((result: ScenarioProgressResult): boolean => result.outcome === 'success')).toBe(true) + const executionEvidence = JSON.stringify({ + source_head: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), + compatibility_id: getCompatibilityId(), scenario_result_id: runId, viewport: page.viewportSize(), + estimate_request: estimateRequest, launch_request: request, saved_configuration: savedConfiguration, progress, + estimate_requests: { + preset_target: targetedResponse.request().postDataJSON(), + edited_selection: selectedResponse.request().postDataJSON(), + held: heldResponse.request().postDataJSON(), + failed: failedResponse.request().postDataJSON(), + refreshed: estimateRequest, + }, + }, null, 2) + const evidencePath = path.join( + process.env.PYRIT_TECHNIQUES_SCREENSHOT_DIR ?? testInfo.outputDir, `${view.name}-execution-evidence.json`, + ) + await writeFile(evidencePath, executionEvidence) + await testInfo.attach('execution-evidence', { path: evidencePath, contentType: 'application/json' }) + await capture(page, testInfo, view.name, 'completed-run') + expect((await readPreset(page, name)).preset).toEqual(saved.preset) + expect(errors).toEqual([]) + }) + + test('uses scenario-relative defaults and explains a forbidden baseline', async ( + { page }: { page: Page }, testInfo: TestInfo, + ): Promise => { + await page.goto('/scanner/launch') + for (const [scenarioName, technique, policy, baseline] of [ + [SCENARIO, 'offline_plain', 'enabled', true], + ['technique_no_baseline', 'offline_rot13', 'forbidden', false], + ['technique_optional_baseline', 'offline_base64', 'disabled', false], + ] as const) { + const metadataResponse = await page.request.get(`/api/scenarios/catalog/${scenarioName}`, { + headers: compatibilityHeaders(), + }) + expect(metadataResponse.status(), await metadataResponse.text()).toBe(200) + const metadata: RegisteredScenario = await metadataResponse.json() + expect(metadata.default_techniques).toEqual([technique]) + expect(metadata.baseline_policy).toBe(policy) + expect(metadata.include_baseline_by_default).toBe(baseline) + await page.getByRole('link', { name: scenarioName, exact: true }).click() + await expect(page).toHaveURL(`/scanner/launch/${scenarioName}`) + await expectSelection(page, metadata.default_techniques, metadata.include_baseline_by_default) + const checkbox = page.getByRole('checkbox', { name: 'baseline', exact: true }) + if (policy === 'forbidden') { + await expect(checkbox).toBeDisabled() + await expect(checkbox).toHaveAccessibleDescription('This scenario does not support a baseline comparison.') + await expect(page.getByText('This scenario does not support a baseline comparison.')).toBeVisible() + await expectUnclipped(page, checkbox) + await capture(page, testInfo, view.name, 'baseline-unavailable') + } else { + await expect(checkbox).toBeEnabled() + await tabTo(page, checkbox) + await expectUnclipped(page, checkbox) + await capture(page, testInfo, view.name, `baseline-${policy}`) + } + await page.getByRole('checkbox', { name: technique, exact: true }).uncheck() + await page.getByRole('checkbox', { + name: technique === 'offline_plain' ? 'offline_base64' : 'offline_plain', exact: true, + }).check() + if (policy !== 'forbidden') await checkbox.setChecked(!baseline) + await page.getByRole('link', { name: 'Back to scanners', exact: true }).click() + } + }) + + test('keeps an unpinned preset tracking scenario defaults after a real description edit', async ( + { page }: { page: Page }, testInfo: TestInfo, + ): Promise => { + const name = `techniques_defaults_${view.name}_${testInfo.retry}` + await createPreset(page, { name, scenario_name: 'technique_optional_baseline' }) + await page.goto(presetEditorRoutePath(name)) + await expectSelection(page, ['offline_base64'], false) + await page.getByRole('textbox', { name: 'Description', exact: true }).fill('Keep scenario-owned defaults.') + await page.getByRole('button', { name: 'Save preset', exact: true }).click() + await expect(page).toHaveURL(PRESETS_ROUTE) + const saved = await readPreset(page, name) + expect(saved.preset.techniques ?? null).toBeNull() + expect(saved.preset.include_baseline ?? null).toBeNull() + await page.goto(presetScannerRoutePath('technique_optional_baseline', name)) + await expectSelection(page, ['offline_base64'], false) + const estimated = nextEstimate(page, 'technique_optional_baseline') + await page.getByRole('combobox', { name: 'Objective Target', exact: true }).selectOption(TARGET) + await readEstimate(await estimated, 1, false) + }) + + test('expands a saved aggregate using the real scenario catalog without selecting unrelated techniques', async ( + { page }: { page: Page }, testInfo: TestInfo, + ): Promise => { + const name = `techniques_aggregate_${view.name}_${testInfo.retry}` + await createPreset(page, { + name, scenario_name: SCENARIO, techniques: ['encoded'], include_baseline: false, + }) + const estimated = nextEstimate(page) + await page.goto(presetScannerRoutePath(SCENARIO, name)) + await expectSelection(page, ['offline_base64', 'offline_rot13'], false) + await readEstimate(await estimated, 2, false) + expect((await readPreset(page, name)).preset.techniques).toEqual(['encoded']) + }) + }) +} diff --git a/frontend/package.json b/frontend/package.json index 72adfbadea..fc558c43c7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -17,6 +17,7 @@ "test:coverage": "jest --coverage", "test:e2e": "playwright test", "test:e2e:scenario-hierarchy": "npm run build && playwright test --config playwright.scenario-hierarchy.config.ts", + "test:e2e:scenario-techniques": "npm run build && playwright test --config playwright.scenario-techniques.config.ts", "test:e2e:analytics": "npm run build && playwright test --config playwright.analytics.config.ts", "test:e2e:seeded": "playwright test --project seeded", "test:e2e:live": "E2E_LIVE_MODE=true playwright test --project live", diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index fe223fad9d..52fbf6032a 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -26,7 +26,7 @@ const E2E_BACKEND_URL = `http://127.0.0.1:${E2E_BACKEND_PORT}`; export default defineConfig({ testDir: "./e2e", - testIgnore: ["**/analytics.spec.ts", "**/scenario-hierarchy.spec.ts"], + testIgnore: ["**/analytics.spec.ts", "**/scenario-hierarchy.spec.ts", "**/scenario-techniques.spec.ts"], fullyParallel: true, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, diff --git a/frontend/playwright.scenario-techniques.config.ts b/frontend/playwright.scenario-techniques.config.ts new file mode 100644 index 0000000000..f8d20c771e --- /dev/null +++ b/frontend/playwright.scenario-techniques.config.ts @@ -0,0 +1,32 @@ +import process from 'node:process' + +import { defineConfig } from '@playwright/test' + +import analytics from './playwright.analytics.config' + +const port = Number(process.env.PYRIT_E2E_TECHNIQUES_PORT ?? '18611') +if (!Number.isInteger(port) || port < 1 || port > 65535) { + throw new Error('PYRIT_E2E_TECHNIQUES_PORT must be a valid TCP port.') +} +const url = `http://127.0.0.1:${port}` + +export default defineConfig({ + ...analytics, + testMatch: 'scenario-techniques.spec.ts', + use: { + ...analytics.use, + baseURL: url, + storageState: { + cookies: [], + origins: [{ origin: url, localStorage: [{ name: 'pyrit-tour-completed', value: 'true' }] }], + }, + }, + projects: [{ name: 'scenario-techniques', use: { viewport: { width: 1440, height: 1050 } } }], + webServer: { + command: `cd .. && uv run --no-sync python -m frontend.e2e.fixtures.scenario_techniques_backend --port ${port}`, + env: { PYTHONIOENCODING: 'utf-8' }, + url: `${url}/api/health`, + reuseExistingServer: false, + timeout: 120_000, + }, +}) diff --git a/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.test.tsx b/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.test.tsx index d8dfccb70f..62eb4ab77e 100644 --- a/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.test.tsx +++ b/frontend/src/components/ScenarioPresets/ScenarioPresetEditor.test.tsx @@ -832,6 +832,17 @@ describe('ScenarioPresetEditor edit mode', () => { expect(mockUpdate.mock.calls.at(-1)?.[1]).not.toHaveProperty('show_in_scanner') }) + it('associates the unavailable baseline with its scenario-owned explanation', async () => { + mockGetScenario.mockResolvedValue(makeScenario({ + baseline_policy: 'forbidden', include_baseline_by_default: false, + })) + renderEdit() + + const baseline = await screen.findByRole('checkbox', { name: 'baseline', exact: true }) + expect(baseline).toBeDisabled() + expect(baseline).toHaveAccessibleDescription('This scenario does not support a baseline comparison.') + }) + it('saves the techniques and baseline choice the operator changed', async () => { const user = userEvent.setup() renderEdit() diff --git a/frontend/src/components/Scenarios/ScenarioDetail.test.tsx b/frontend/src/components/Scenarios/ScenarioDetail.test.tsx index 88230fb7d8..29f1019b44 100644 --- a/frontend/src/components/Scenarios/ScenarioDetail.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioDetail.test.tsx @@ -1081,6 +1081,35 @@ describe('ScenarioDetail', () => { }) }) + it('keeps preview actions keyboard accessible without changing the selected techniques or baseline', async () => { + const user = userEvent.setup() + renderDetail('/scanner/launch/foundry.red_team_agent') + const baseline = await screen.findByRole('checkbox', { name: 'baseline', exact: true }) + await user.click(baseline) + const launch = screen.getByRole('button', { name: 'Launch scan', exact: true }) + await user.click(launch) + const preview = await screen.findByRole('dialog', { name: 'Run preview' }) + const cancel = within(preview).getByRole('button', { name: 'Cancel', exact: true }) + await waitFor(() => expect(cancel).toHaveFocus()) + await user.tab() + expect(within(preview).getByRole('button', { name: 'Launch scan', exact: true })).toHaveFocus() + await user.tab({ shift: true }) + expect(cancel).toHaveFocus() + await user.keyboard(' ') + expect(await screen.findByRole('checkbox', { name: 'default_technique', exact: true })).toBeChecked() + expect(screen.getByRole('checkbox', { name: 'baseline', exact: true })).not.toBeChecked() + await waitFor(() => expect(launch).toHaveFocus()) + await user.keyboard('{Enter}') + const reopened = await screen.findByRole('dialog', { name: 'Run preview' }) + await waitFor(() => expect(within(reopened).getByRole('button', { name: 'Cancel', exact: true })).toHaveFocus()) + await user.tab() + expect(within(reopened).getByRole('button', { name: 'Launch scan', exact: true })).toHaveFocus() + await user.keyboard(' ') + await waitFor(() => expect(mockStartRun).toHaveBeenCalledWith(expect.objectContaining({ + techniques: ['default_technique'], include_baseline: false, + }))) + }) + it('restores focus to Launch scan button when preview dialog is dismissed via Escape', async () => { const user = userEvent.setup() renderDetail('/scanner/launch/foundry.red_team_agent') @@ -1192,12 +1221,13 @@ describe('ScenarioDetail', () => { expect(within(screen.getByTestId('run-estimate')).getByText('8')).toBeInTheDocument() await user.selectOptions(screen.getByTestId('scenario-target-select'), 'target-b') + expect(within(screen.getByTestId('run-estimate')).getByRole('status')).toHaveTextContent('Updating estimate...') await advanceTimers(300) await flushRenderedPromises() const estimate = screen.getByTestId('run-estimate') expect(within(estimate).getByText('8')).toBeInTheDocument() - expect(within(estimate).getByText('Preview service unavailable')).toBeInTheDocument() + expect(within(estimate).getByRole('status')).toHaveTextContent('Preview service unavailable') expect(screen.getByTestId('scenario-target-select')).toHaveValue('target-b') expect(screen.getByTestId('launch-scenario-btn')).not.toBeDisabled() }) @@ -1551,10 +1581,15 @@ describe('ScenarioDetail', () => { renderDetail('/scanner/launch/foundry.red_team_agent') await screen.findByRole('checkbox', { name: 'crescendo' }) expect(screen.getAllByRole('button', { name: 'Select Multi-turn techniques' })[0]).toBeDisabled() + expect(screen.getAllByRole('button', { name: 'Select Multi-turn techniques' })[0]) + .toHaveAccessibleDescription(/at most 2.*Clear a selected technique/) await user.click(screen.getByRole('checkbox', { name: 'crescendo' })) expect(screen.getByRole('checkbox', { name: 'many_shot' })).toBeDisabled() + expect(screen.getByRole('checkbox', { name: 'many_shot' })) + .toHaveAccessibleDescription(/at most 2.*Clear a selected technique/) expect(screen.getByRole('checkbox', { name: 'baseline' })).toBeEnabled() await user.click(screen.getByRole('checkbox', { name: 'default_technique' })) + expect(screen.getByRole('checkbox', { name: 'many_shot' })).not.toHaveAttribute('aria-describedby') await user.click(screen.getAllByRole('button', { name: 'Select Multi-turn techniques' })[0]) expect(screen.getByRole('checkbox', { name: 'crescendo' })).toBeChecked() expect(screen.getByRole('checkbox', { name: 'many_shot' })).toBeChecked() @@ -1748,6 +1783,7 @@ describe('ScenarioDetail', () => { const checkbox = screen.getByTestId('baseline-checkbox') expect(checkbox).toBeDisabled() expect(checkbox).not.toBeChecked() + expect(checkbox).toHaveAccessibleDescription('This scenario does not support a baseline comparison.') await confirmRunPreview(user) await waitFor(() => expect(mockStartRun).toHaveBeenCalled()) diff --git a/frontend/src/components/Scenarios/ScenarioDetail.tsx b/frontend/src/components/Scenarios/ScenarioDetail.tsx index f3ed6c9b60..448b9445d4 100644 --- a/frontend/src/components/Scenarios/ScenarioDetail.tsx +++ b/frontend/src/components/Scenarios/ScenarioDetail.tsx @@ -764,6 +764,8 @@ function ScenarioLaunchForm({ }: ScenarioLaunchFormProps) { const identityStyles = useRegistryIdentityRowStyles() const identityId = useId() + const techniqueLimitDescriptionId = `${identityId}-technique-limit` + const baselineUnavailableId = `${identityId}-baseline-unavailable` const [expandedIdentities, setExpandedIdentities] = useState>(new Set()) const runtime = useRuntime() const { rememberScanTargets } = useUserPreferences() @@ -1484,6 +1486,12 @@ function ScenarioLaunchForm({ ? 'Select exactly one technique. Baseline comparison is optional and separate.' : 'Select individual techniques, or use a tag to select or clear all techniques with that tag.'} } + {supportsTechniques && !singleTechnique && scenario.max_concrete_techniques != null && ( + + Select at most {scenario.max_concrete_techniques} concrete techniques. + {' '}Clear a selected technique or choose a smaller tag. + + )} {techniqueSelectionError && ( {techniqueSelectionError} @@ -1521,6 +1529,9 @@ function ScenarioLaunchForm({
{selectableTechniques.map((technique) => { const selected = isTechniqueSelected(technique) + const limited = !technique.isBaseline && !selected + && scenario.max_concrete_techniques != null + && selectedTechniques.length >= scenario.max_concrete_techniques const expanded = expandedIdentities.has(technique.name) const regionId = `${identityId}-${encodeURIComponent(technique.name)}` return ( @@ -1540,9 +1551,9 @@ function ScenarioLaunchForm({ className={styles.selectionControl} label={technique.name} checked={selected} - disabled={submitting || technique.disabled || (!technique.isBaseline && !selected - && scenario.max_concrete_techniques != null - && selectedTechniques.length >= scenario.max_concrete_techniques)} + disabled={submitting || technique.disabled || limited} + aria-describedby={technique.disabled ? baselineUnavailableId + : limited ? techniqueLimitDescriptionId : undefined} onChange={(_, data) => handleTechniqueChange(technique, data.checked === true)} data-testid={technique.isBaseline ? 'baseline-checkbox' : `technique-${technique.name}`} />} @@ -1573,6 +1584,8 @@ function ScenarioLaunchForm({ appearance="outline" checked={tagSelected} disabled={submitting || tagMembers.length === 0 || (!tagSelected && exceedsLimit)} + aria-describedby={tagMembers.length === 0 ? baselineUnavailableId + : !tagSelected && exceedsLimit ? techniqueLimitDescriptionId : undefined} onClick={() => handleTagChange(tag)} aria-label={`${tagSelected ? 'Clear' : 'Select'} ${techniqueSetName(tag)} techniques`} > @@ -1583,7 +1596,7 @@ function ScenarioLaunchForm({
)} {technique.disabled && ( - + This scenario does not support a baseline comparison. )} @@ -1816,13 +1829,13 @@ function ScenarioLaunchForm({ )} {estimateState.status === 'refreshing' && ( -
+
Updating estimate...
)} {estimateState.status === 'stale' && ( - + {estimateState.error} )} @@ -2026,6 +2039,7 @@ function ScenarioLaunchForm({