Skip to content

Refine the summary page to make the main result clear #52

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kentwelcome merged 1 commit into from
Oct 21, 2026
Merged

Refine the summary page to make the main result clear #52

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 74 additions & 37 deletions .agents/skills/run-behavior-diff-human-evaluation/scripts/quiz.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -401,7 +401,7 @@ def _attention_shape(body):
children = _children(body)
if (
len(children) < 3
or children[0].tag != "h3"
or children[0].tag != "h4"
or _plain(children[0]) != "What needs your attention"
or children[1].tag != "p"
or not children[1].has_class("attention-assessment")
Expand Down Expand Up @@ -531,7 +531,7 @@ def _blinded_report(source):
):
raise ValueError("Unknown summary story structure.")
steps = _children(story_children[1])
names = ("finding", "expectation", "method", "evidence", "next")
names = ("evidence", "finding", "expectation", "method")
if len(steps) != len(names) or any(
step.tag != "li"
or not step.has_class("story-step")
Expand All @@ -543,34 +543,59 @@ def _blinded_report(source):
_shape(step, [("span", "story-number"), ("div", "story-body")])[1]
for step in steps
]
# Check digest and qualification summarize the assessment; drop them but
# accept only these exact optional siblings before the observations.
evidence_shape = [("h3", None)]
for tag, optional in (
("div", "summary-check-digest"),
("p", "summary-qualification"),
):
if any(node.has_class(optional) for node in _children(bodies[3])):
evidence_shape.append((tag, optional))
evidence_shape.extend([("div", "summary-observations"), ("nav", "evidence-nav")])
evidence_parts = _shape(bodies[3], evidence_shape)
evidence_heading, observations = evidence_parts[0], evidence_parts[-2]
evidence_heading, observations, _ = _shape(
bodies[0],
[("h3", None), ("div", "summary-observations"), ("div", "summary-expected")],
)
children = _children(observations)
shape = [("p", "summary-provenance"), ("div", "summary-pair")]
if any(node.has_class("primary-result-context") for node in children):
shape.append(("section", "primary-result-context"))
shape.append(
_shape(observations, shape)
scope_shape = [("h3", None)]
for tag, optional in (("p", "summary-qualification"), ("div", "summary-caution")):
if any(node.has_class(optional) for node in _children(bodies[1])):
scope_shape.append((tag, optional))
scope_shape.extend([("ul", "summary-notices"), ("div", "summary-attention")])
scope = _shape(bodies[1], scope_shape)
attention = scope[-1]
context = _shape(
bodies[3],
[("h3", None), ("details", "summary-context-details"), ("nav", "evidence-nav")],
)[1]
context_children = _children(context)
context_shape = [("summary", None), ("div", "summary-method-grid")]
context_shape.append(
("p", "summary-interpretation-unavailable")
if any(
node.has_class("summary-interpretation-unavailable") for node in children
node.has_class("summary-interpretation-unavailable")
for node in context_children
)
else ("div", "summary-why")
)
for optional in ("summary-case-context", "summary-caution"):
if any(node.has_class(optional) for node in children):
shape.append(("div", optional))
shape.append(("div", "summary-attention"))
_shape(observations, shape)
if any(node.has_class("summary-case-context") for node in context_children):
context_shape.append(("div", "summary-case-context"))
context_shape.append(("div", "summary-start-context"))
if any(node.has_class("summary-other-changes") for node in context_children):
context_shape.append(("p", "summary-other-changes"))
context_shape.append(("details", "summary-reading-help"))
_shape(context, context_shape)
retained = (
[
node
for node in context_children
if any(
node.has_class(name)
for name in (
"summary-why",
"summary-interpretation-unavailable",
"summary-case-context",
)
)
]
+ [node for node in scope if node.has_class("summary-caution")]
+ [attention]
)
pair = children[1]
cards = _shape(
pair,
Expand All @@ -582,23 +607,30 @@ def _blinded_report(source):
)
for card in (cards[0], cards[2]):
parts = _children(card)
start = bool(len(parts) > 1 and parts[1].has_class("summary-start"))
empty = bool(parts and parts[-1].has_class("summary-empty"))
card_shape = [("h4", "summary-side-label")]
if start:
card_shape.append(("div", "summary-start"))
card_shape.extend(
_shape(
card,
[
("svg", "summary-picture"),
("h4", "summary-side-label"),
("p", "summary-empty") if empty else ("ul", "summary-choices"),
]
],
)
starts = _single(
[node for node in context_children if node.has_class("summary-start-context")],
"comparison starts",
)
for start in _children(starts):
if start.tag != "div" or not start.has_class("summary-start"):
raise ValueError("Unknown comparison start.")
_shape(
start,
[
("strong", None),
("p", "summary-start-text"),
("p", "summary-start-source"),
],
)
_shape(card, card_shape)
if start:
start_shape = [("p", "summary-start-text"), ("p", "summary-start-source")]
if len(_children(parts[1])) == 3:
start_shape.append(("span", "summary-start-arrow"))
_shape(parts[1], start_shape)
retained.insert(0, starts)
source_intents = [
node
for node in nodes
Expand All @@ -607,12 +639,17 @@ def _blinded_report(source):
for name in ("purpose-goal-text", "edit-aim", "story-intent")
)
]
for node in children:
for node in retained:
if node.has_class("summary-case-context"):
_case_context(node, source_intents)
elif node.has_class("summary-why") or node.has_class("summary-caution"):
_claim_shape(node)
_attention_shape(children[-1])
_attention_shape(attention)
observations = _Node(
observations.tag,
observations.attrs.items(),
[*observations.children, *retained],
)
# Keep only the bounded observation section. Assessment verdicts, expected
# criteria, preparation rationale and next actions can reveal the keyed edit.
trimmed_story = _Node(
Expand All @@ -630,7 +667,7 @@ def _blinded_report(source):
[
_Node(
"li",
steps[3].attrs.items(),
steps[0].attrs.items(),
[
_Node(
"div",
Expand Down
44 changes: 25 additions & 19 deletions docs/architecture.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ scripts handle execution, evidence, and reporting.
+---------------------------------------------------------+
| Presentation |
| render.py -> consistent HTML, Markdown, structured JSON |
| Five-step Summary; detailed tabs retain evidence |
| Observation-first Summary; details retain evidence |
+----------------------------+----------------------------+
v
+---------------------------------------------------------+
Expand Down Expand Up @@ -558,19 +558,26 @@ semantic truth, causal attribution, or live extractor adherence.
| `report-data.json` | Structured, versioned report data. |
| `report-artifact.html` | Embeddable HTML body. |

The default Summary has five numbered steps in one main reading column:
finding; expected versus observed behavior; how tested; supporting evidence;
and next action. A separate validated-result badge precedes a concrete heading
from the lead extracted comparison's topic and status. An unmet requirement keeps
the canonical finding prominent; a missing comparison uses that finding as its
fallback. The comparison heading is not a claim that the change's goal was met.
The first step pairs the recorded aim with observed choices and counts, then
identifies unmet, unresolved and uncovered requirements without inventing findings.
Full checks and original observation detail remain below; technical rejection
reasons stay in diagnostics rather than dominating the first screen.
A compact header keeps source/PR links, compared revisions and the model visible
without a hover. Reading help, complete task, raw setup limits and rejected-check
diagnostics expand inline. The other five tabs retain their navigation roles.
The default Summary has four numbered sections in one reading column:
observed Before/After; scope and attention; what the tests establish; and
expandable test context and interpretation. Compact cards retain every branch
and count, without decorative illustrations. The recorded aim follows the cards.
A separate validated-result badge precedes the existing comparison heading;
an unmet requirement retains canonical finding precedence. Neither a changed
comparison nor a missing assessment is a pass of the instruction's goals.
Material cautions, report-wide qualifications, incomplete evidence, and attention
findings stay visible. The canonical finding, grouped check states, and every
unmet check's full title precede one disclosure containing the full check table,
rejection diagnostics, and unresolved or uncovered requirements. Next-action
guidance remains visible. Setup, full task, model interpretation, and comparison
starts expand separately. HTML and Markdown use the same hierarchy and evidence
destinations; the other five tabs retain their content and navigation roles.

The blind human-evaluation exporter validates this four-section DOM and projects
only observations, primary-result context, comparison starts, interpretation,
cautions, and attention evidence. Aim, assessment, setup, and next-action
guidance remain excluded. Unknown layouts fail closed rather than using a
legacy-layout fallback; saved already-built quiz artifacts remain unchanged.

The assessment conclusion is derived from validated target checks, not narrative prose.
An unmet After requirement remains unmet when Before made the same mistake.
Expand Down Expand Up @@ -648,17 +655,16 @@ Explanation validation additionally checks bounded narrative, row citations,
unchanged-row status, and exact example excerpts; it does not prove semantic
interpretation truth. Legacy comparison ranking is only a fallback when no validated
narrative selects an observed behavior.
When target assessment is available, task-completion context stays in the detailed
comparisons rather than adding another Summary block; mixed primary results still
appear in the Summary's evidence-limit notices.
When the selected lead differs from the canonical primary result, primary status
and full distributions remain beside the cards even with a target assessment.
Changed explanations, citations, or presentation must not be described as changed
actions. When a fallback comparison lead is not the primary task result,
`content.primary_result_context` exposes the primary status and full
distribution beside those cards in both formats. Missing or incomplete evidence
cannot become an unchanged-result claim. The primary-result block remains
derived presentation, not a separate serialized report field.
The primary-result block is context, not a second navigation choice.
**Understand the change**, under the Summary's evidence section, opens a new
**Understand the change**, linked from the Summary's context section, opens the
top-level explanation tab. This middle layer explains the specific distinction,
its practical implications, important unchanged behavior, consistency and
minorities, and concrete limits, with links to decision rows and named trials.
Expand All @@ -673,7 +679,7 @@ instead of inferring a no-difference claim or making a new model call.
The general change explanation retains its completeness guard; incomplete
records do not erase supported target assessments of other completed trials.
Target coverage names unassessed records separately from observed failures.
Plans are not presented as executions. The third Summary section gives concise
Plans are not presented as executions. The second Summary section gives concise
consequences, actions, and essential evidence limits with one section-level detail link.
Findings linked to unmet After criteria come first. If no finding covers an
unmet criterion, a deterministic item retains its outcome distribution and
Expand Down
Loading
Loading