diff --git a/.agents/skills/run-behavior-diff-human-evaluation/scripts/quiz.py b/.agents/skills/run-behavior-diff-human-evaluation/scripts/quiz.py index 1e4162f..2e87cc7 100644 --- a/.agents/skills/run-behavior-diff-human-evaluation/scripts/quiz.py +++ b/.agents/skills/run-behavior-diff-human-evaluation/scripts/quiz.py @@ -401,7 +401,7 @@ def _attention_shape(body): children = _children(body) if ( len(children) < 3 - or children[0].tag != "h3" + or children[0].tag != "h4" or _plain(children[0]) != "What needs your attention" or children[1].tag != "p" or not children[1].has_class("attention-assessment") @@ -531,7 +531,7 @@ def _blinded_report(source): ): raise ValueError("Unknown summary story structure.") steps = _children(story_children[1]) - names = ("finding", "expectation", "method", "evidence", "next") + names = ("evidence", "finding", "expectation", "method") if len(steps) != len(names) or any( step.tag != "li" or not step.has_class("story-step") @@ -543,34 +543,59 @@ def _blinded_report(source): _shape(step, [("span", "story-number"), ("div", "story-body")])[1] for step in steps ] - # Check digest and qualification summarize the assessment; drop them but - # accept only these exact optional siblings before the observations. - evidence_shape = [("h3", None)] - for tag, optional in ( - ("div", "summary-check-digest"), - ("p", "summary-qualification"), - ): - if any(node.has_class(optional) for node in _children(bodies[3])): - evidence_shape.append((tag, optional)) - evidence_shape.extend([("div", "summary-observations"), ("nav", "evidence-nav")]) - evidence_parts = _shape(bodies[3], evidence_shape) - evidence_heading, observations = evidence_parts[0], evidence_parts[-2] + evidence_heading, observations, _ = _shape( + bodies[0], + [("h3", None), ("div", "summary-observations"), ("div", "summary-expected")], + ) children = _children(observations) shape = [("p", "summary-provenance"), ("div", "summary-pair")] if any(node.has_class("primary-result-context") for node in children): shape.append(("section", "primary-result-context")) - shape.append( + _shape(observations, shape) + scope_shape = [("h3", None)] + for tag, optional in (("p", "summary-qualification"), ("div", "summary-caution")): + if any(node.has_class(optional) for node in _children(bodies[1])): + scope_shape.append((tag, optional)) + scope_shape.extend([("ul", "summary-notices"), ("div", "summary-attention")]) + scope = _shape(bodies[1], scope_shape) + attention = scope[-1] + context = _shape( + bodies[3], + [("h3", None), ("details", "summary-context-details"), ("nav", "evidence-nav")], + )[1] + context_children = _children(context) + context_shape = [("summary", None), ("div", "summary-method-grid")] + context_shape.append( ("p", "summary-interpretation-unavailable") if any( - node.has_class("summary-interpretation-unavailable") for node in children + node.has_class("summary-interpretation-unavailable") + for node in context_children ) else ("div", "summary-why") ) - for optional in ("summary-case-context", "summary-caution"): - if any(node.has_class(optional) for node in children): - shape.append(("div", optional)) - shape.append(("div", "summary-attention")) - _shape(observations, shape) + if any(node.has_class("summary-case-context") for node in context_children): + context_shape.append(("div", "summary-case-context")) + context_shape.append(("div", "summary-start-context")) + if any(node.has_class("summary-other-changes") for node in context_children): + context_shape.append(("p", "summary-other-changes")) + context_shape.append(("details", "summary-reading-help")) + _shape(context, context_shape) + retained = ( + [ + node + for node in context_children + if any( + node.has_class(name) + for name in ( + "summary-why", + "summary-interpretation-unavailable", + "summary-case-context", + ) + ) + ] + + [node for node in scope if node.has_class("summary-caution")] + + [attention] + ) pair = children[1] cards = _shape( pair, @@ -582,23 +607,30 @@ def _blinded_report(source): ) for card in (cards[0], cards[2]): parts = _children(card) - start = bool(len(parts) > 1 and parts[1].has_class("summary-start")) empty = bool(parts and parts[-1].has_class("summary-empty")) - card_shape = [("h4", "summary-side-label")] - if start: - card_shape.append(("div", "summary-start")) - card_shape.extend( + _shape( + card, [ - ("svg", "summary-picture"), + ("h4", "summary-side-label"), ("p", "summary-empty") if empty else ("ul", "summary-choices"), - ] + ], + ) + starts = _single( + [node for node in context_children if node.has_class("summary-start-context")], + "comparison starts", + ) + for start in _children(starts): + if start.tag != "div" or not start.has_class("summary-start"): + raise ValueError("Unknown comparison start.") + _shape( + start, + [ + ("strong", None), + ("p", "summary-start-text"), + ("p", "summary-start-source"), + ], ) - _shape(card, card_shape) - if start: - start_shape = [("p", "summary-start-text"), ("p", "summary-start-source")] - if len(_children(parts[1])) == 3: - start_shape.append(("span", "summary-start-arrow")) - _shape(parts[1], start_shape) + retained.insert(0, starts) source_intents = [ node for node in nodes @@ -607,12 +639,17 @@ def _blinded_report(source): for name in ("purpose-goal-text", "edit-aim", "story-intent") ) ] - for node in children: + for node in retained: if node.has_class("summary-case-context"): _case_context(node, source_intents) elif node.has_class("summary-why") or node.has_class("summary-caution"): _claim_shape(node) - _attention_shape(children[-1]) + _attention_shape(attention) + observations = _Node( + observations.tag, + observations.attrs.items(), + [*observations.children, *retained], + ) # Keep only the bounded observation section. Assessment verdicts, expected # criteria, preparation rationale and next actions can reveal the keyed edit. trimmed_story = _Node( @@ -630,7 +667,7 @@ def _blinded_report(source): [ _Node( "li", - steps[3].attrs.items(), + steps[0].attrs.items(), [ _Node( "div", diff --git a/docs/architecture.md b/docs/architecture.md index 86a5bff..f68b00a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -49,7 +49,7 @@ scripts handle execution, evidence, and reporting. +---------------------------------------------------------+ | Presentation | | render.py -> consistent HTML, Markdown, structured JSON | -| Five-step Summary; detailed tabs retain evidence | +| Observation-first Summary; details retain evidence | +----------------------------+----------------------------+ v +---------------------------------------------------------+ @@ -558,19 +558,26 @@ semantic truth, causal attribution, or live extractor adherence. | `report-data.json` | Structured, versioned report data. | | `report-artifact.html` | Embeddable HTML body. | -The default Summary has five numbered steps in one main reading column: -finding; expected versus observed behavior; how tested; supporting evidence; -and next action. A separate validated-result badge precedes a concrete heading -from the lead extracted comparison's topic and status. An unmet requirement keeps -the canonical finding prominent; a missing comparison uses that finding as its -fallback. The comparison heading is not a claim that the change's goal was met. -The first step pairs the recorded aim with observed choices and counts, then -identifies unmet, unresolved and uncovered requirements without inventing findings. -Full checks and original observation detail remain below; technical rejection -reasons stay in diagnostics rather than dominating the first screen. -A compact header keeps source/PR links, compared revisions and the model visible -without a hover. Reading help, complete task, raw setup limits and rejected-check -diagnostics expand inline. The other five tabs retain their navigation roles. +The default Summary has four numbered sections in one reading column: +observed Before/After; scope and attention; what the tests establish; and +expandable test context and interpretation. Compact cards retain every branch +and count, without decorative illustrations. The recorded aim follows the cards. +A separate validated-result badge precedes the existing comparison heading; +an unmet requirement retains canonical finding precedence. Neither a changed +comparison nor a missing assessment is a pass of the instruction's goals. +Material cautions, report-wide qualifications, incomplete evidence, and attention +findings stay visible. The canonical finding, grouped check states, and every +unmet check's full title precede one disclosure containing the full check table, +rejection diagnostics, and unresolved or uncovered requirements. Next-action +guidance remains visible. Setup, full task, model interpretation, and comparison +starts expand separately. HTML and Markdown use the same hierarchy and evidence +destinations; the other five tabs retain their content and navigation roles. + +The blind human-evaluation exporter validates this four-section DOM and projects +only observations, primary-result context, comparison starts, interpretation, +cautions, and attention evidence. Aim, assessment, setup, and next-action +guidance remain excluded. Unknown layouts fail closed rather than using a +legacy-layout fallback; saved already-built quiz artifacts remain unchanged. The assessment conclusion is derived from validated target checks, not narrative prose. An unmet After requirement remains unmet when Before made the same mistake. @@ -648,9 +655,8 @@ Explanation validation additionally checks bounded narrative, row citations, unchanged-row status, and exact example excerpts; it does not prove semantic interpretation truth. Legacy comparison ranking is only a fallback when no validated narrative selects an observed behavior. -When target assessment is available, task-completion context stays in the detailed -comparisons rather than adding another Summary block; mixed primary results still -appear in the Summary's evidence-limit notices. +When the selected lead differs from the canonical primary result, primary status +and full distributions remain beside the cards even with a target assessment. Changed explanations, citations, or presentation must not be described as changed actions. When a fallback comparison lead is not the primary task result, `content.primary_result_context` exposes the primary status and full @@ -658,7 +664,7 @@ distribution beside those cards in both formats. Missing or incomplete evidence cannot become an unchanged-result claim. The primary-result block remains derived presentation, not a separate serialized report field. The primary-result block is context, not a second navigation choice. -**Understand the change**, under the Summary's evidence section, opens a new +**Understand the change**, linked from the Summary's context section, opens the top-level explanation tab. This middle layer explains the specific distinction, its practical implications, important unchanged behavior, consistency and minorities, and concrete limits, with links to decision rows and named trials. @@ -673,7 +679,7 @@ instead of inferring a no-difference claim or making a new model call. The general change explanation retains its completeness guard; incomplete records do not erase supported target assessments of other completed trials. Target coverage names unassessed records separately from observed failures. -Plans are not presented as executions. The third Summary section gives concise +Plans are not presented as executions. The second Summary section gives concise consequences, actions, and essential evidence limits with one section-level detail link. Findings linked to unmet After criteria come first. If no finding covers an unmet criterion, a deterministic item retains its outcome distribution and diff --git a/plugin/skills/behavior-diff/scripts/reporting/content.py b/plugin/skills/behavior-diff/scripts/reporting/content.py index d33fc78..29ca3ae 100644 --- a/plugin/skills/behavior-diff/scripts/reporting/content.py +++ b/plugin/skills/behavior-diff/scripts/reporting/content.py @@ -364,11 +364,12 @@ def primary_result_context(report): ) -SUMMARY_FINDING_HEADING = "What we found" -SUMMARY_EXPECTATION_HEADING = "What was expected, and what happened" -SUMMARY_METHOD_HEADING = "How we tested it" -SUMMARY_EVIDENCE_HEADING = "What the evidence shows" -SUMMARY_NEXT_HEADING = "What to do next" +SUMMARY_FINDING_HEADING = "Scope and attention" +SUMMARY_EXPECTATION_HEADING = "What these tests establish" +SUMMARY_METHOD_HEADING = "Test context and interpretation" +SUMMARY_EVIDENCE_HEADING = "Observed Before / After" +SUMMARY_CHECK_DETAILS_LABEL = "All checks, rejection reasons, and unresolved questions" +SUMMARY_CONTEXT_DETAILS_LABEL = "Setup, full task, and model interpretation" SUMMARY_CONTEXT_LABEL = "Context for this comparison" SUMMARY_MEANING_LABEL = "Model interpretation" SUMMARY_ATTENTION_HEADING = "What needs your attention" @@ -419,18 +420,16 @@ def required_evidence_text(check): "The heading names the lead extracted comparison and whether it differs " "between Before and After. A difference is not evidence that the edit's aim " "was achieved.", - "The badge and Step 1 give the result, decided only from the validated checks " - "of the goals recorded before trials and from trial completeness. Step 1 also " - "shows the edit's aim, the lead observed change, and what remains unknown or " - "unmet.", - "Step 2 lists each expected result with its Before and After results and says " - "whether it was recorded before trials or written by the assessor afterward. " - "Unclear, unassessable, and not-assessed trials stay separate from Yes and No.", - "Step 3 shows the actual task, the scenario and test-data rationale recorded " - "before trials, and the setup. Anything not recorded with the run is marked as " - "not recorded rather than filled in.", - "Step 4 shows the observed behavior; model interpretations are labeled.", - "Step 5 suggests a next step. The report never approves or blocks a change.", + "Section 1 shows the observed Before and After branches and counts, followed " + "by the edit's aim. Section 2 keeps material uncertainty and attention visible.", + "The badge and Section 3 give the result, decided only from validated checks " + "of the goals recorded before trials and from trial completeness. Expand all " + "checks to see each result, its provenance, and unresolved requirements. " + "Unclear, unassessable, and not-assessed trials stay separate from Yes and No. " + "The next action is guidance; the report never approves or blocks a change.", + "Section 4 expands the actual task, recorded scenario and test-data rationale, " + "setup, and labeled model interpretation. Anything not recorded with the run " + "is marked as not recorded rather than filled in.", ) @@ -449,13 +448,10 @@ def task_preview(task): SUMMARY_RATIONALE_HEADING = "Why this test data challenges the aim" SUMMARY_CONDITIONS_HEADING = "Test conditions" SUMMARY_INSPECTED_HEADING = "What we inspected" -SUMMARY_CAVEATS_HEADING = "What this does not prove" SUMMARY_AIM_LABEL = "What the edit aimed at" -SUMMARY_CHANGE_LABEL = "Observed change" SUMMARY_OTHER_CHANGES_LABEL = "Other comparisons that differ" SUMMARY_OPEN_LABEL = "Still unknown or unmet" SUMMARY_AIM_UNCHECKED = "Whether the edit achieved the aim above" -SUMMARY_IMPLICATION_LABEL = "What this means:" SUMMARY_DIAGNOSTICS_LABEL = "Assessment diagnostic" SUMMARY_DIGEST_HEADING = "Results by check" @@ -604,7 +600,7 @@ def summary_heading(report): def summary_result(report): - """Return (result text, diagnostic or None) for Step 1. + """Return the canonical result text and any assessment diagnostic. The canonical finding is repeated only when the heading does not already show it. Without validated checks, the assessment's technical reason is diagnostic. @@ -616,46 +612,14 @@ def summary_result(report): return " ".join(parts), summary.meaning -@dataclass(frozen=True) -class SummaryChange: - scope: str - sides: Tuple[Tuple[str, str], ...] - note: str - others: Tuple[str, ...] - - -def summary_change(report): - """The lead extracted comparison, stated once, from extracted choices only.""" - summary = report.summary - row = _lead_row(report) - if row is None: - return None - sides = tuple( - ( - label, - "; ".join( - f"{choice.choice} ({trial_count(choice.count, data.total)})" - for choice in data.choices - ) - or NO_EXTRACTED_CHOICE, - ) - for label, data in (("Before", summary.before), ("After", summary.after)) - ) - others = ( - tuple( - _row_name(other) - for index, other in enumerate(report.decisions.rows, 1) - if index != summary.decision and decision_status(other) == "Changed" - ) - if complete_trial_evidence(report) - else () - ) - return SummaryChange( - f"Lead comparison, {source_label(row.anchor, report.metadata.trace_source)}:" - f" {_row_name(row)}", - sides, - "Trial evidence is incomplete." if summary.status == "unavailable" else "", - others, +def summary_other_changes(report): + """Other changed comparisons, without repeating the lead's cards.""" + if _lead_row(report) is None or not complete_trial_evidence(report): + return () + return tuple( + _row_name(row) + for index, row in enumerate(report.decisions.rows, 1) + if index != report.summary.decision and decision_status(row) == "Changed" ) @@ -736,62 +700,13 @@ def summary_check_groups(report): ) -def _goal_gap(report): - unchecked = unchecked_goals(report.decisions) - if not unchecked: - return "" - noun = "goal" if len(unchecked) == 1 else "goals" - return f"No check covers recorded {noun} " + ", ".join(map(str, unchecked)) + "." - - -def summary_implication(report): - """What the validated finding allows a reader to conclude, decided from states.""" - summary = report.summary - checks = summary.checks - state = summary.state - if state == "unavailable": - return ( - "No conclusion about the expected behavior is available. A difference " - "in the observations is not evidence that the expected behavior was achieved." - ) +def summary_check_link_label(report, state, number): + """Name unmet checks in the default view; other checks retain their numbers.""" if state == "not_met": - return ( - "At least one expected behavior was not met in these tests; other " - "supported results do not offset it." - ) - if state == "inconclusive": - settled = _settled(checks) - unsettled = sum(check.state == "inconclusive" for check in checks) - rejected = sum(check.state == "unavailable" for check in checks) - gap = _goal_gap(report) - parts = [] - if unsettled: - parts.append( - f"{unsettled} {'check is' if unsettled == 1 else 'checks are'} not settled." - ) - if rejected: - parts.append( - f"{rejected} {'check was' if rejected == 1 else 'checks were'} not " - "assessed because validation rejected them." - ) - if gap: - parts.append(gap) - lead = ( - f"{settled} of {len(checks)} checks have a supported result, but this " - "is not a pass of every expected behavior." - if settled - else "No check reached a supported result." + return next( + check.text for check in report.summary.checks if check.criterion == number ) - return " ".join((lead, *parts)) - if state == "baseline_incomplete": - return ( - "After met every check, but incomplete Before results mean no " - "improvement is shown." - ) - return ( - "Every check has a supported result in this scenario. This does not show " - "reliability in other situations or that the edit caused the difference." - ) + return f"Check {number}" def summary_qualification(report): @@ -802,7 +717,7 @@ def summary_qualification(report): "comparisons; they are retained as written, not endorsed." ) if summary.state == "unavailable": - # A rejection's technical reason stays in the Step 1 diagnostic. + # A rejection's technical reason stays in the full-checks diagnostic. return ( f"{lead} Because no validated check of the expected behavior is " "available, any claim of success or verification in them is unconfirmed." diff --git a/plugin/skills/behavior-diff/scripts/reporting/render_html.py b/plugin/skills/behavior-diff/scripts/reporting/render_html.py index fda4ed7..b682ad3 100644 --- a/plugin/skills/behavior-diff/scripts/reporting/render_html.py +++ b/plugin/skills/behavior-diff/scripts/reporting/render_html.py @@ -766,16 +766,12 @@ def _summary_start(report, side: str) -> str: start = content.summary_start(report, side) if start is None: return "" - arrow = ( - '' - if start.ordered - else "" - ) return ( '
' + f"{side.capitalize()} observed start" f'

{version_name_html(start.text)}

' f'

{html.escape(start.source)}

' - f"{arrow}
" + "" ) @@ -801,13 +797,13 @@ def _summary_side(report, side: str) -> str: f'
' f'

{label}

' - f"{_summary_start(report, side)}{illustration(data.icon)}{body}
" + f"{body}" ) def _primary_result_context(report) -> str: context = content.primary_result_context(report) - if context is None or report.decisions.target_assessment is not None: + if context is None: return "" sides = "".join( f"

{html.escape(label)}: {html.escape(text)}

" @@ -1127,31 +1123,16 @@ def _summary_step(number: int, css_class: str, heading: str, body: str) -> str: ) -def _summary_change(report) -> str: - change = content.summary_change(report) - if change is None: - return f'

{html.escape(report.summary.evidence_label)}

' - sides = "".join( - f"
  • {label}: {html.escape(text)}
    • " - for label, text in change.sides - ) - note = ( - f'

    {html.escape(change.note)}

    ' - if change.note - else "" - ) - others = ( +def _summary_other_changes(report) -> str: + others = content.summary_other_changes(report) + return ( '

    ' f"{html.escape(content.SUMMARY_OTHER_CHANGES_LABEL)}: " - f"{html.escape(', '.join(change.others))} " + f"{html.escape(', '.join(others))} " 'Behavior diff

    ' - if change.others + if others else "" ) - return ( - f'

    {html.escape(change.scope)}

    ' - f'{note}{others}' - ) def _summary_open(report) -> str: @@ -1252,7 +1233,8 @@ def _short_story_summary(report: ReportData) -> str: + "".join( f'
  • {html.escape(label)} ' + ", ".join( - f'Check {number}' + f'' + f"{html.escape(content.summary_check_link_label(report, state, number))}" for number in numbers ) + "
    • " @@ -1271,10 +1253,16 @@ def _short_story_summary(report: ReportData) -> str: steps = ( _summary_step( 1, - "story-finding", - content.SUMMARY_FINDING_HEADING, - f"{result_html}" - '
    ' + "story-evidence", + content.SUMMARY_EVIDENCE_HEADING, + '
    ' + f'

    {html.escape(summary.evidence_label)}

    ' + '
    ' + f"{_summary_side(report, 'before')}" + '' + f"{_summary_side(report, 'after')}
    " + f"{_primary_result_context(report)}
    " + '
    ' f'

    {html.escape(content.SUMMARY_AIM_LABEL)}

    ' '
    ' f'{html.escape(intent.badge)}' @@ -1282,57 +1270,46 @@ def _short_story_summary(report: ReportData) -> str: '
    ' - f'

    {intent_text}

    ' - '
    ' - f'

    {html.escape(content.SUMMARY_CHANGE_LABEL)}

    ' - f"{_summary_change(report)}
    " - f"{_summary_open(report)}{diagnostic_html}", + f'

    {intent_text}

    ', ), _summary_step( 2, + "story-finding", + content.SUMMARY_FINDING_HEADING, + f"{qualification}{caution}" + f'' + '
    ' + f"

    {html.escape(content.SUMMARY_ATTENTION_HEADING)}

    " + f"{_attention_summary(report)}
    ", + ), + _summary_step( + 3, "story-expectation", content.SUMMARY_EXPECTATION_HEADING, - f'

    {html.escape(content.SUMMARY_IMPLICATION_LABEL)}' - f" {html.escape(content.summary_implication(report))}

    " - f"{_summary_checks(report)}", + f"{result_html}{digest}" + '
    ' + f"{html.escape(content.SUMMARY_CHECK_DETAILS_LABEL)}" + f"{_summary_checks(report)}{_summary_open(report)}{diagnostic_html}
    " + f'

    {version_name_html(summary.next_action)}

    ' + f'

    {html.escape(GATE_NOTE)}

    ', ), _summary_step( - 3, + 4, "story-method", content.SUMMARY_METHOD_HEADING, + '
    ' + f"{html.escape(content.SUMMARY_CONTEXT_DETAILS_LABEL)}" f'
    {"".join(method_parts)}
    ' - '
    ' - f"

    {html.escape(content.SUMMARY_CAVEATS_HEADING)}

    " - f'
    {reading_help}', - ), - _summary_step( - 4, - "story-evidence", - content.SUMMARY_EVIDENCE_HEADING, - f"{digest}{qualification}" - '
    ' - f'

    {html.escape(summary.evidence_label)}

    ' - '
    ' - f"{_summary_side(report, 'before')}" - '' - f"{_summary_side(report, 'after')}
    " - f"{_primary_result_context(report)}{meaning}{case_context}{caution}" - '
    ' - f"

    {html.escape(content.SUMMARY_ATTENTION_HEADING)}

    " - f"{_attention_summary(report)}
    " + f"{meaning}{case_context}" + '
    ' + f"{_summary_start(report, 'before')}{_summary_start(report, 'after')}
    " + f"{_summary_other_changes(report)}{reading_help}
    " '', ), - _summary_step( - 5, - "story-next", - content.SUMMARY_NEXT_HEADING, - f'

    {version_name_html(summary.next_action)}

    ' - f'

    {html.escape(GATE_NOTE)}

    ', - ), ) return ( '
    ' diff --git a/plugin/skills/behavior-diff/scripts/reporting/render_markdown.py b/plugin/skills/behavior-diff/scripts/reporting/render_markdown.py index ce11d8e..40ded0f 100644 --- a/plugin/skills/behavior-diff/scripts/reporting/render_markdown.py +++ b/plugin/skills/behavior-diff/scripts/reporting/render_markdown.py @@ -556,32 +556,64 @@ def _summary_markdown(report): "## Summary\n", f"`{_text(content.summary_status(report))}`\n", f"**{_text(content.summary_heading(report))}**\n", - f"### 1. {content.SUMMARY_FINDING_HEADING}\n", + f"### 1. {content.SUMMARY_EVIDENCE_HEADING}\n", + _text(summary.evidence_label) + "\n", ] - if result_text: - markdown.append(_text(result_text) + "\n") + for side_name in ("before", "after"): + side = getattr(summary, side_name) + markdown.append(f"#### {side_name.capitalize()}\n") + if not side.choices: + markdown.append("No supported comparison is available.\n") + for choice in side.choices: + markdown.append(f"**{_text(choice.label)}**\n") + if choice.detail: + markdown.append(_text(choice.detail) + "\n") + markdown.append(_text(content.trial_count(choice.count, side.total)) + "\n") + context = content.primary_result_context(report) + if context is not None: + markdown.append(f"#### {_text(context.heading)} — {_text(context.status)}\n") + for label, text in context.sides: + markdown.append(f"**{label}:** {_text(text)}\n") + markdown.append(_text(context.note) + "\n") markdown += [ f"#### {content.SUMMARY_AIM_LABEL}\n", f"**{_text(intent.label)}**\n", _text(intent.text) + "\n", _text(intent.note) + "\n", "[Instruction changes →](#instruction-diff)\n", - f"#### {content.SUMMARY_CHANGE_LABEL}\n", + f"### 2. {content.SUMMARY_FINDING_HEADING}\n", ] - change = content.summary_change(report) - if change is None: - markdown.append(_text(summary.evidence_label) + "\n") - else: - markdown.append(_text(change.scope) + "\n") - markdown += [f"- **{label}:** {_text(text)}" for label, text in change.sides] - markdown.append("") - if change.note: - markdown.append(_text(change.note) + "\n") - if change.others: - markdown.append( - f"**{content.SUMMARY_OTHER_CHANGES_LABEL}:** " - f"{_text(', '.join(change.others))} [Behavior diff →](#panel-decision)\n" + qualification = content.summary_qualification(report) + if qualification: + markdown.append(f"> {_text(qualification)}\n") + if summary.caution: + markdown.append( + f"**Watch out:** {_text(summary.caution.text)} " + f"{_decision_links(summary.caution.decisions)}\n" + ) + markdown += [f"- {_text(notice)}" for notice in summary.notices] + markdown.append("") + markdown += _attention_summary_markdown(report) + markdown.append(f"### 3. {content.SUMMARY_EXPECTATION_HEADING}\n") + if result_text: + markdown.append(_text(result_text) + "\n") + groups = content.summary_check_groups(report) + if groups: + markdown.append(f"#### {content.SUMMARY_DIGEST_HEADING}\n") + markdown += [ + f"- **{_text(label)}** " + + ", ".join( + f"[{_text(content.summary_check_link_label(report, state, number))}]" + f"(#{content.target_criterion_anchor(number)})" + for number in numbers ) + for state, label, numbers in groups + ] + markdown.append("") + markdown.append( + f"
    {content.SUMMARY_CHECK_DETAILS_LABEL}\n" + ) + markdown += _summary_checks_markdown(report) open_items = content.summary_open_items(report) if open_items: markdown.append(f"#### {content.SUMMARY_OPEN_LABEL}\n") @@ -597,18 +629,17 @@ def _summary_markdown(report): markdown.append("") if diagnostic: markdown += [ - f"
    {content.SUMMARY_DIAGNOSTICS_LABEL}\n", + f"#### {content.SUMMARY_DIAGNOSTICS_LABEL}\n", _text(diagnostic) + "\n", - "
    \n", ] markdown += [ - f"### 2. {content.SUMMARY_EXPECTATION_HEADING}\n", - f"**{content.SUMMARY_IMPLICATION_LABEL}** " - f"{_text(content.summary_implication(report))}\n", + "
    \n", + _text(summary.next_action) + "\n", + _text(GATE_NOTE) + "\n", + f"### 4. {content.SUMMARY_METHOD_HEADING}\n", + f"
    {content.SUMMARY_CONTEXT_DETAILS_LABEL}\n", + f"#### {content.SUMMARY_TASK_HEADING}\n", ] - markdown += _summary_checks_markdown(report) - markdown.append(f"### 3. {content.SUMMARY_METHOD_HEADING}\n") - markdown.append(f"#### {content.SUMMARY_TASK_HEADING}\n") if report.content.task: markdown += [ f"**{content.SUMMARY_TASK_LABEL}:** " @@ -623,52 +654,6 @@ def _summary_markdown(report): markdown.append(f"#### {heading}\n") markdown += [f"- {_text(line)}" for line in lines] markdown.append("") - markdown.append(f"#### {content.SUMMARY_CAVEATS_HEADING}\n") - markdown += [f"- {_text(notice)}" for notice in summary.notices] - markdown += ["", f"
    {content.SUMMARY_HELP_LABEL}\n"] - markdown += [f"- {_text(line)}" for line in content.SUMMARY_HELP] - markdown += ["", "
    \n"] - markdown += ["", f"### 4. {content.SUMMARY_EVIDENCE_HEADING}\n"] - groups = content.summary_check_groups(report) - if groups: - markdown.append(f"#### {content.SUMMARY_DIGEST_HEADING}\n") - markdown += [ - f"- **{_text(label)}** " - + ", ".join( - f"[Check {number}](#{content.target_criterion_anchor(number)})" - for number in numbers - ) - for _, label, numbers in groups - ] - markdown.append("") - qualification = content.summary_qualification(report) - if qualification: - markdown.append(f"> {_text(qualification)}\n") - markdown.append(_text(summary.evidence_label) + "\n") - for side_name in ("before", "after"): - side = getattr(summary, side_name) - markdown.append(f"#### {side_name.capitalize()}\n") - start = content.summary_start(report, side_name) - if start is not None: - markdown += [ - f"**Observed start:** {_text(start.text)}\n", - _text(start.source) + "\n", - ] - if start.ordered: - markdown.append("↓\n") - if not side.choices: - markdown.append("No supported comparison is available.\n") - for choice in side.choices: - markdown.append(f"**{_text(choice.label)}**\n") - if choice.detail: - markdown.append(_text(choice.detail) + "\n") - markdown.append(_text(content.trial_count(choice.count, side.total)) + "\n") - context = content.primary_result_context(report) - if context is not None and report.decisions.target_assessment is None: - markdown.append(f"#### {_text(context.heading)} — {_text(context.status)}\n") - for label, text in context.sides: - markdown.append(f"**{label}:** {_text(text)}\n") - markdown.append(_text(context.note) + "\n") if summary.why is None: markdown.append(content.SUMMARY_INTERPRETATION_UNAVAILABLE + "\n") else: @@ -681,21 +666,26 @@ def _summary_markdown(report): f"**{content.SUMMARY_CONTEXT_LABEL}:** {_text(summary.context.text)} " f"{_decision_links(summary.context.decisions)}\n" ) - if summary.caution: + for side_name in ("before", "after"): + start = content.summary_start(report, side_name) + if start is not None: + markdown += [ + f"**{side_name.capitalize()} observed start:** {_text(start.text)}\n", + _text(start.source) + "\n", + ] + others = content.summary_other_changes(report) + if others: markdown.append( - f"**Watch out:** {_text(summary.caution.text)} " - f"{_decision_links(summary.caution.decisions)}\n" + f"**{content.SUMMARY_OTHER_CHANGES_LABEL}:** " + f"{_text(', '.join(others))} [Behavior diff →](#panel-decision)\n" ) - markdown += _attention_summary_markdown(report) + markdown += [f"
    {content.SUMMARY_HELP_LABEL}\n"] + markdown += [f"- {_text(line)}" for line in content.SUMMARY_HELP] + markdown += ["", "
    \n", "
    \n"] markdown.append( "[Understand the change →](#panel-explanation) · " "[Behavior diff →](#panel-decision) · [Trial evidence →](#panel-trials)\n" ) - markdown += [ - f"### 5. {content.SUMMARY_NEXT_HEADING}\n", - _text(summary.next_action) + "\n", - _text(GATE_NOTE) + "\n", - ] return markdown diff --git a/plugin/skills/behavior-diff/scripts/reporting/report.css b/plugin/skills/behavior-diff/scripts/reporting/report.css index 6c3eca2..fd50922 100644 --- a/plugin/skills/behavior-diff/scripts/reporting/report.css +++ b/plugin/skills/behavior-diff/scripts/reporting/report.css @@ -442,7 +442,7 @@ details { margin-bottom:.25rem; } .summary-pair { display:grid; grid-template-columns:minmax(0, 1fr) 36px minmax(0, 1fr); align-items:stretch; gap:10px; margin:16px 0; } .summary-card { background:#fff; border:1px solid #dbe5e8; border-radius:16px; - padding:16px 18px 20px; text-align:center; min-width:0; } + padding:14px 18px; text-align:left; min-width:0; } .summary-card.summary-after { background:#fffaf0; border-color:#ead8b5; } .summary-side-label { font-size:12px; text-transform:uppercase; font-weight:750; letter-spacing:.12em; margin:0 0 4px; } @@ -454,7 +454,7 @@ details { margin-bottom:.25rem; } .summary-choice h5 { font-size:22px; letter-spacing:-.025em; color:#142b35; margin:3px 0 8px; line-height:1.3; } .summary-choice p, .summary-empty { font-size:14px; line-height:1.5; color:#53646d; - max-width:300px; margin:0 auto 14px; } + margin:0 0 10px; } .summary-count { display:block; font-size:12px; font-weight:700; color:#142b35; font-variant-numeric:tabular-nums; } .summary-arrow { align-self:center; text-align:center; font-size:30px; color:#758790; } @@ -462,7 +462,6 @@ details { margin-bottom:.25rem; } .summary-start-text { border:1px solid currentColor; border-radius:8px; padding:10px 12px; margin:0; font-size:15px; line-height:1.4; font-weight:650; } .summary-start-source { color:#53646d; font-size:11px; line-height:1.5; margin:6px 0 0; } -.summary-start-arrow { display:block; font-size:28px; line-height:1.2; margin:4px 0 0; } .summary-case-context { border-left:3px solid #137780; background:#edf4f5; border-radius:0 6px 6px 0; padding:12px 18px; margin:16px 0 0; max-width:780px; } .summary-case-context > strong { font-size:13px; } @@ -485,18 +484,12 @@ details { margin-bottom:.25rem; } .summary-diagnostics { margin-top:10px; font-size:13px; color:#53646d; } .summary-diagnostics > summary { font-weight:650; } .summary-diagnostics p { margin:6px 0 0; line-height:1.6; } -.summary-expect-observe { display:grid; grid-template-columns:minmax(0, 1fr) minmax(0, 1fr); - border:1px solid #dbe5e8; border-radius:10px; overflow:hidden; } -.summary-expected, .summary-observed { padding:16px 18px; min-width:0; } -.summary-expected { background:#f6f9f8; } -.summary-observed { background:#fffaf0; border-left:1px solid #dbe5e8; } +.summary-expected { padding:16px 18px; min-width:0; background:#f6f9f8; + border:1px solid #dbe5e8; border-radius:10px; } .summary-panel-label { margin:0 0 8px; color:#53646d; letter-spacing:.11em; font-size:11px; font-weight:750; text-transform:uppercase; } .summary-expected .story-intent { font-size:18px; } -.summary-change-scope { margin:0 0 6px; font-size:13px; color:#53646d; font-weight:650; } -.summary-change-sides { margin:0; padding-left:18px; font-size:16px; line-height:1.55; } -.summary-change-sides li + li { margin-top:4px; } -.summary-change-note, .summary-other-changes { margin:10px 0 0; font-size:14px; +.summary-other-changes { margin:10px 0 0; font-size:14px; line-height:1.55; } .summary-open { margin:16px 0 0; max-width:900px; } .summary-open h4 { font-size:14px; margin:0 0 8px; } @@ -512,14 +505,16 @@ details { margin-bottom:.25rem; } .expected-checks { margin:18px 0 0; } .expected-checks-label { margin:0; font-size:14px; } .expected-checks-note { margin:4px 0 0; max-width:780px; } -.summary-implication { font-size:15px; line-height:1.6; margin:0; max-width:780px; } .summary-method-grid { display:grid; grid-template-columns:minmax(0, 1fr) minmax(0, 1fr); gap:16px 24px; } -.summary-method-part, .summary-method-caveats { min-width:0; } -.summary-method-part h4, .summary-method-caveats h4, .summary-check-digest h4 { +.summary-method-part { min-width:0; } +.summary-method-part h4, .summary-check-digest h4 { font-size:14px; margin:0 0 6px; } .summary-method-list { font-size:14px; line-height:1.7; margin:0; padding-left:18px; } -.summary-method-caveats { margin-top:16px; padding-top:12px; border-top:1px solid #dbe5e8; } +.summary-check-details, .summary-context-details { margin:16px 0; } +.summary-check-details > summary, .summary-context-details > summary { + padding:10px 0; font-weight:650; cursor:pointer; } +.summary-start-context { display:flex; gap:16px; flex-wrap:wrap; } .summary-check-digest ul { list-style:none; margin:0 0 14px; padding:0; font-size:14px; line-height:1.7; } .summary-check-digest a { margin-left:4px; } @@ -553,7 +548,6 @@ details { margin-bottom:.25rem; } .summary-notices { font-size:13px; color:#53646d; margin:16px 0 0; padding-left:18px; line-height:1.7; } .summary-notices:empty { display:none; } -.story-method .summary-notices { margin-top:0; } .summary-evidence-button { display:inline-flex; align-items:center; gap:13px; font-size:13px; font-weight:700; background:#173e4a; color:#fff; border-radius:9px; padding:11px 15px !important; text-decoration:none; } @@ -579,9 +573,8 @@ details { margin-bottom:.25rem; } .story-body > h3, .intent-heading h3 { font-size:15px; } .story-body > h3, .intent-heading { margin-top:4px; } .story-intent, .summary-expected .story-intent { font-size:17px; } - .summary-expect-observe, .summary-method-grid { grid-template-columns:minmax(0, 1fr); } - .summary-observed { border-left:0; border-top:1px solid #dbe5e8; } - .summary-expected, .summary-observed { padding:14px 12px; } + .summary-method-grid { grid-template-columns:minmax(0, 1fr); } + .summary-expected { padding:14px 12px; } .summary-checks { min-width:760px; } .summary-checks th, .summary-checks td { overflow-wrap:normal; } .summary-context { font-size:16px; } @@ -592,7 +585,7 @@ details { margin-bottom:.25rem; } .summary-card { padding:14px 12px; } .summary-picture { width:110px; height:78px; margin:4px auto; } .summary-choice h5 { font-size:20px; margin:7px 0; } - .summary-choice p { margin:0 auto 10px; font-size:13px; } + .summary-choice p { margin:0 0 10px; font-size:13px; } .summary-why, .summary-caution { display:block; } .summary-why p { margin-top:7px; font-size:16px; } .summary-caution p { margin-top:5px; } diff --git a/tests/human-evaluation-quiz-test.py b/tests/human-evaluation-quiz-test.py index 827d5ed..5247c06 100755 --- a/tests/human-evaluation-quiz-test.py +++ b/tests/human-evaluation-quiz-test.py @@ -486,10 +486,10 @@ def test_assessment_setup_and_next_actions_do_not_reveal_quiz_key(self): root = quiz._ReportParser().finish(source) hidden = ( "summary-headline", - "story-finding", - "story-expectation", - "story-method", - "story-next", + "summary-expected", + "summary-check-details", + "summary-method-grid", + "summary-next-action", ) secrets = [] for index, name in enumerate(hidden): @@ -499,11 +499,7 @@ def test_assessment_setup_and_next_actions_do_not_reveal_quiz_key(self): ) secret = f"PRIVATE-ANSWER-KEY-{index}" secrets.append(secret) - if name == "summary-headline": - node.children.append(secret) - else: - body = quiz._children(node)[1] - body.children.append(quiz._Node("p", children=[secret])) + node.children.append(secret) with patch.object(quiz._ReportParser, "finish", return_value=root): blinded = quiz._blinded_report(source) for secret in secrets: @@ -543,7 +539,7 @@ def test_old_summary_shapes_and_wrapper_recovery_are_rejected(self): child for child in steps.children if not isinstance(child, quiz._Node) - or not child.has_class("story-next") + or not child.has_class("story-method") ] elif legacy == "unknown-step": quiz._children(steps)[3].attrs["class"] = "story-step" @@ -590,7 +586,7 @@ def decorated(extra=None): ) body = quiz._children( quiz._single( - [n for n in nodes if n.has_class("story-evidence")], "evidence" + [n for n in nodes if n.has_class("story-finding")], "scope" ) )[1] status = quiz._children(story)[0] @@ -599,15 +595,9 @@ def decorated(extra=None): parts = [ node for node in quiz._children(body) - if not node.has_class("summary-check-digest") - and not node.has_class("summary-qualification") + if not node.has_class("summary-qualification") ] added = [ - quiz._Node( - "div", - [("class", "summary-check-digest")], - [quiz._Node("h4", children=["LEAKDIGEST checks"])], - ), quiz._Node( "p", [("class", "summary-qualification")], @@ -621,7 +611,7 @@ def decorated(extra=None): with patch.object(quiz._ReportParser, "finish", return_value=decorated()): blinded = quiz._blinded_report(source) - for leak in ("LEAKSTATUS", "LEAKDIGEST", "LEAKQUALIFICATION"): + for leak in ("LEAKSTATUS", "LEAKQUALIFICATION"): self.assertNotIn(leak, blinded) self.assertIn("summary-observations", blinded) self.assertIn("evidence-limits", blinded) @@ -981,7 +971,7 @@ def test_attention_structure_and_svg_resource_boundaries_fail_closed(self): 1, ) ) - picture_end = source.index("", source.index('class="summary-pair"')) + picture_end = source.index("", source.index('class="summary-choice"')) for markup in ( "

    Arbitrary markup

    ", '', @@ -989,7 +979,11 @@ def test_attention_structure_and_svg_resource_boundaries_fail_closed(self): ): with self.subTest(markup=markup), self.assertRaises(ValueError): quiz._blinded_report( - source[:picture_end] + markup + source[picture_end:] + source[:picture_end] + + "" + + markup + + "" + + source[picture_end:] ) for value in ( "url(https://hostile.invalid/resource)", diff --git a/tests/presentation_contract.py b/tests/presentation_contract.py index 56c102e..83d0ab8 100644 --- a/tests/presentation_contract.py +++ b/tests/presentation_contract.py @@ -13,34 +13,100 @@ def _summary_links(report): - """Expected per-step links: aim to the diff, open goals and checks to criteria, - other differing comparisons and evidence to detail tabs.""" + """Observation links to instructions; coverage and context retain evidence.""" from reporting import content - change = content.summary_change(report) return [ - ["#instruction-diff"] - + (["#panel-decision"] if change is not None and change.others else []) - + [ - f"#target-criterion-{item.criterion}" - for item in content.summary_open_items(report) - if item.criterion is not None - ], - [f"#target-criterion-{check.criterion}" for check in report.summary.checks], - [], + ["#instruction-diff"], + ["#attention-explanation"], [ f"#target-criterion-{number}" for _, _, numbers in content.summary_check_groups(report) for number in numbers ] + + [f"#target-criterion-{check.criterion}" for check in report.summary.checks] + [ - "#attention-explanation", - "#panel-explanation", - "#panel-decision", - "#panel-trials", + f"#target-criterion-{item.criterion}" + for item in content.summary_open_items(report) + if item.criterion is not None ], - [], + (["#panel-decision"] if content.summary_other_changes(report) else []) + + ["#panel-explanation", "#panel-decision", "#panel-trials"], + ] + + +def _assert_observation_first(report, page): + from reporting import content + from reporting.render_html import render_document + from reporting.render_markdown import render_markdown + + stored = report.to_json() + markup = _ReportMarkup(page) + summary = markup.select(anchor="panel-summary")[0] + assert ( + _ReportMarkup(render_document(page)).select(anchor="panel-summary")[0] + == summary + ) + status = markup.select(summary, class_name="summary-status")[0] + assert status["attrs"]["data-state"] == report.summary.state + assert _plain(status) == content.summary_status(report) + steps = markup.select(summary, class_name="story-step") + assert [step["attrs"]["class"] for step in steps] == [ + "story-step story-evidence", + "story-step story-finding", + "story-step story-expectation", + "story-step story-method", ] + disclosures = markup.select(summary, tag="details") + hidden = {id(node) for detail in disclosures for node in markup.nodes(detail)} + cards = markup.select(steps[0], class_name="summary-card") + assert len(cards) == 2 + assert all(not markup.select(card, tag="svg") for card in cards) + provenance = markup.select(steps[0], class_name="summary-provenance")[0] + assert _plain(provenance) == report.summary.evidence_label + for card, side in zip(cards, (report.summary.before, report.summary.after)): + assert id(card) not in hidden + choices = markup.select(card, class_name="summary-choice") + assert len(choices) == len(side.choices) + for node, choice in zip(choices, side.choices): + assert choice.label in _plain(node) + assert content.trial_count(choice.count, side.total) in _plain(node) + details = markup.select(summary, class_name="summary-check-details")[0] + assert "open" not in details["attrs"] + assert len(markup.select(details, class_name="summary-check")) == ( + len(report.summary.checks) + len(content.expected_checks(report).unassessed) + if content.expected_checks(report) + else 0 + ) + digest = markup.select(summary, class_name="summary-check-digest") + for check in report.summary.checks: + if check.state == "not_met": + assert digest and id(digest[0]) not in hidden + assert check.text in _plain(digest[0]) + for cls in ( + "summary-caution", + "summary-qualification", + "summary-notices", + "story-attention", + ): + for node in markup.select(summary, class_name=cls): + assert id(node) not in hidden + qualification = content.summary_qualification(report) + if qualification: + assert _plain(summary).count(qualification) == 1 + md = html.unescape(render_markdown(report)).replace("\\", "") + headings = [ + content.SUMMARY_EVIDENCE_HEADING, + content.SUMMARY_FINDING_HEADING, + content.SUMMARY_EXPECTATION_HEADING, + content.SUMMARY_METHOD_HEADING, + ] + assert [ + md.index(f"### {n}. {title}") for n, title in enumerate(headings, 1) + ] == sorted(md.index(f"### {n}. {title}") for n, title in enumerate(headings, 1)) + assert report.summary.headline in md + assert report.summary.evidence_label in md + assert report.to_json() == stored # {B} and {A} mark the only words that name a compared version. @@ -317,7 +383,9 @@ def assert_summary_detail_links(reports): from reporting.render_html import render_artifact for name, report in reports.items(): - markup = _ReportMarkup(render_artifact(report, "")) + page = render_artifact(report, "") + _assert_observation_first(report, page) + markup = _ReportMarkup(page) summary = markup.select(anchor="panel-summary")[0] steps = markup.select(summary, class_name="story-step") expected = _summary_links(report) diff --git a/tests/report-schema-test.py b/tests/report-schema-test.py index 2b7a75a..24155ec 100755 --- a/tests/report-schema-test.py +++ b/tests/report-schema-test.py @@ -489,8 +489,8 @@ def assert_scenario_record_contract(raw): page = render_artifact(frozen, "") markdown = render_markdown(frozen) expectation = _between(page, "story-expectation", "story-method") - method = _between(page, "story-method", "story-evidence") - evidence = _between(page, "story-evidence", "story-next") + method = _between(page, "story-method", "evidence-nav") + evidence = _between(page, "story-evidence", "story-finding") assert content.FROZEN_CHECKS_LABEL in expectation assert content.ASSESSOR_CHECKS_LABEL not in page assert content.FROZEN_UNASSESSED_MEANING not in page @@ -503,7 +503,7 @@ def assert_scenario_record_contract(raw): ): assert heading in method and html.escape(text) in method, heading assert heading in markdown, heading - # The blinded quiz keeps Step 4; preparation text must stay outside it. + # Preparation text stays outside the observed comparison. assert html.escape(text) not in evidence, heading assert content.SCENARIO_UNRECORDED not in page assert content.FROZEN_CHECKS_LABEL in markdown @@ -607,7 +607,7 @@ def assert_scenario_record_contract(raw): ) assert content.FROZEN_CHECKS_LABEL not in page assert html.escape(content.SCENARIO_UNRECORDED) in _between( - page, "story-method", "story-evidence" + page, "story-method", "evidence-nav" ) assert content.ASSESSOR_CHECKS_LABEL in render_markdown(legacy) unrecorded = ReportData.from_dict(raw) @@ -1271,6 +1271,9 @@ def assessed(before="met", after="met", mode=None, variants=None, drop=False): assert_round_trip(value.to_dict()) # Every export carries the same derived conclusion and per-check results. pages = (render_artifact(value, ""), render_markdown(value)) + from presentation_contract import _assert_observation_first + + _assert_observation_first(value, pages[0]) for page in map(visible, pages): for text in ( summary.headline, @@ -1333,7 +1336,7 @@ def _focus_markup(page): summary = markup.select(anchor="panel-summary")[0] status = markup.select(summary, class_name="summary-status")[0] heading = markup.select(summary, class_name="summary-headline")[0] - step = markup.select(summary, class_name="story-finding")[0] + step = markup.select(summary, class_name="story-expectation")[0] kinds = [ entry["attrs"].get("data-kind") for listing in markup.select(step, class_name="summary-open-list") @@ -1403,10 +1406,12 @@ def check(value): assert summary.state == "unavailable" and not summary.checks assert content.decision_status(row) == "Changed" assert row.topic in heading and heading != summary.headline - change = content.summary_change(unavailable) - assert change is not None and row.topic in change.scope - for (_, side_text), side in zip(change.sides, (row.before, row.after)): - assert all(choice.choice in side_text for choice in side) + from attention_contract import _ReportMarkup, _plain + + markup = _ReportMarkup(render_artifact(unavailable, "")) + cards = markup.select(class_name="summary-card") + for card, side in zip(cards, (summary.before, summary.after)): + assert all(choice.label in _plain(card) for choice in side.choices) assert diagnostic == summary.meaning and summary.meaning not in text assert items and {item.kind for item in items} == {"unchecked"} assert all(item.criterion is None for item in items) @@ -1460,7 +1465,7 @@ def check(value): summary, heading, text, diagnostic, items = check(bare) assert summary.state == "unavailable" assert heading == summary.headline - assert content.summary_change(bare) is None + assert content.summary_other_changes(bare) == () assert [item.text for item in items] == [content.SUMMARY_AIM_UNCHECKED] @@ -2420,14 +2425,12 @@ def with_primary(row=original, **changes): "", 1 )[0] markdown = render_markdown(report) - md_block = markdown.split(f"#### {context.heading}", 1)[1].split("### 3.", 1)[0] + md_block = markdown.split(f"#### {context.heading}", 1)[1].split("### 2.", 1)[0] assert expected_status in block and expected_status in md_block assert "