Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions services/harness/src/compiler/evidence.py
Original file line number Diff line number Diff line change
Expand Up @@ -1755,7 +1755,7 @@ def _prices_words(unit: Unit) -> str:


_WORD_CHAR = re.compile(r"[^\W_]", re.UNICODE)
_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])")
_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z])(?=[A-Z])|(?<=[A-Z0-9])(?=[A-Z][a-z])")

# Unicode general categories for a combining mark: nonspacing (Mn), spacing
# combining (Mc), and enclosing (Me). Python's `re` has no `\p{M}` class, so
Expand Down Expand Up @@ -1851,8 +1851,11 @@ def _tokens(text: str) -> tuple[str, ...]:
"WorldDevelopmentIndicators" as `("world", "development",
"indicators")`, the same tokens spaces would have given, without
collapsing the name to one blob a short, unrelated word could contain a
substring of. Matching is Unicode-aware in both directions: Latin accents
are folded before splitting, so "Český" and "Cesky" tokenize the same,
substring of. A digit and the capital after it split only where the
capital starts a word ("Covid19Deaths"), so "DAC2A" and "DAC2a", one
table's name in two cases, are the one token `dac2a`. Matching is
Unicode-aware in both directions: Latin accents are folded before
splitting, so "Český" and "Cesky" tokenize the same,
while a mark on a non-Latin base is left alone (recomposed with it into
one code point by NFC where composition allows, kept as base-plus-mark by
`_words` where it does not), so "が" and "か", or "क़" and "क", keep
Expand Down
19 changes: 14 additions & 5 deletions services/harness/src/planner/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@
from translator.discovery import (
DefinitionNotResolvedError,
PartNotResolvedError,
load_log,
resolve_definition,
resolved_spec,
)
Expand All @@ -90,7 +91,6 @@
Decline,
EvidenceRequirement,
FullHistory,
PinSource,
Plan,
PlanStatus,
SessionState,
Expand Down Expand Up @@ -1219,9 +1219,18 @@ def _first_year_wanted(result: Mapping[str, object]) -> str | None:
FIRST_YEAR_READ_DEADLINE_SECONDS = 10.0


def _first_year_identity(pin: str, shown: Mapping[str, PinSource]) -> Mapping[str, Any] | None:
"""The identity a first-year read of `pin` reads; `None` when none is on record."""
recorded = identity_on_record(pin, shown=shown, end_open=True)
def _first_year_identity(pin: str, state: SessionState) -> Mapping[str, Any] | None:
"""The identity a first-year read of `pin` reads; `None` when none is on record.

It is on record as `submission.service.admit` finds it, the discovery
log's definitions included, so a pin `propose_plan` admits is read.
"""
recorded = identity_on_record(
pin,
shown=pin_sources(state),
end_open=True,
definition_lookup=load_log(state).recorded_definition,
)
return recorded.identity if recorded is not None else None


Expand Down Expand Up @@ -1459,7 +1468,7 @@ async def propose_plan_reading(**arguments: Any) -> dict[str, object]:
pin = _first_year_wanted(result)
if pin is None or pin in full_histories(state):
return result
identity = _first_year_identity(pin, pin_sources(state))
identity = _first_year_identity(pin, state)
if identity is None:
return result
try:
Expand Down
12 changes: 9 additions & 3 deletions services/harness/src/submission/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -411,7 +411,7 @@ def admit(
previous, evidence_requirements, removed_evidence_requirements
)
read = full_histories(view.state)
definition_lookup = load_log(view.state).recorded_alternative
definition_lookup = load_log(view.state).recorded_definition
settled = [
*settled,
*_user_settled_rewrites(
Expand Down Expand Up @@ -2678,7 +2678,10 @@ def identity_on_record(
"""The definition on record for `pin`; `None` when none is.

The definition is the one a search in the conversation recorded
(`planner.plan.recorded_identity`). For a requirement stating
(`planner.plan.recorded_identity`), else the one `definition_lookup`
answers: `admit` passes the discovery log's
(`translator.discovery.DiscoveryLog.recorded_definition`), which holds
a candidate the translator offered. For a requirement stating
`period_end_open` it is otherwise the one the block the planner was
shown carries (`planner.plan.shown_identity`): the translator can find
a closed requirement's pin by searching its stated years, and no search
Expand Down Expand Up @@ -2741,7 +2744,10 @@ def _window_pins(
`scope.window` would mint), and so is the decision that fixes the pin.

The identity on record is the one a search in the conversation recorded
(`planner.plan.recorded_identity`). One of a `latest` window is copied
(`planner.plan.recorded_identity`), else the one `definition_lookup`
answers, the discovery log's
(`translator.discovery.DiscoveryLog.recorded_definition`) under
`admit`. One of a `latest` window is copied
like any other: the copy replaces the window whole, as `get_data`'s
scope surgery does, and the translator confirms the copy through Search
before it is retrieved.
Expand Down
19 changes: 14 additions & 5 deletions services/harness/src/translator/discovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -624,15 +624,23 @@ def recorded_source(self, spec_id: str) -> SelectedSource | None:
return found
return None

def recorded_alternative(self, spec_id: str) -> _RecordedSourceDefinition | None:
"""A verified sibling definition offered by a source in this session."""
def recorded_definition(self, spec_id: str) -> _RecordedSourceDefinition | None:
"""The verified definition of `spec_id` a source this log holds states, or `None`.

The source is `spec_id`'s own, such as a candidate a
`pinned_source_not_found` referral offered, or one offering it as a
sibling measure (`elected_measure.alternatives`). Its identity must
hash to `spec_id` (`caller_definition`).
"""
for own in reversed(tuple(self.sources.values())):
for source in reversed(tuple(own.values())):
identities = (
{a.spec_id: a.identity for a in source.elected_measure.alternatives}
if source.elected_measure is not None
else {}
)
if source.identity is not None:
identities[source.spec_id] = source.identity
identity = identities.get(spec_id)
if identity is None:
continue
Expand Down Expand Up @@ -2186,8 +2194,9 @@ async def _recorded_source(

The identity is the copy over the requirement's period that
`planner.agent.propose_plan` pinned (`planner.plan.windowed_pins`), or
the one a search in the conversation recorded for the pin itself
(`planner.plan.recorded_identity`) when its window is that period, or
the one a search in the conversation or this session's discovery
recorded for the pin itself (`planner.plan.recorded_identity`,
`DiscoveryLog.recorded_definition`) when its window is that period, or
a range with no end year for a requirement stating `period_end_open`,
from `first_year` where the plan reads its dataset's whole history
(`planner.plan.retrieves_period`). It
Expand All @@ -2206,7 +2215,7 @@ async def _recorded_source(
recorded = (
windowed_pins(tool_context.state).get(spec_id)
or recorded_identity(spec_id)
or load_log(tool_context.state).recorded_alternative(spec_id)
or load_log(tool_context.state).recorded_definition(spec_id)
)
identity = recorded.identity if recorded is not None else None
if (
Expand Down
18 changes: 18 additions & 0 deletions services/harness/tests/compiler/test_evidence.py
Original file line number Diff line number Diff line change
Expand Up @@ -820,6 +820,24 @@ def test_a_stated_source_matches_a_provenance_label_split_at_an_acronym_boundary
assert errors(issues) == []


@pytest.mark.parametrize(
("stated", "label"),
[("OECD DAC2a", "OECD DAC2A"), ("Covid19 Deaths", "Covid19Deaths")],
ids=["table-letter-in-either-case", "digit-before-a-word"],
)
def test_a_stated_source_matches_a_provenance_label_with_a_digit_before_a_capital(
requirement: RequirementFactory, source: SourceFactory, stated: str, label: str
) -> None:
# Search labels the DAC2a provenance "OECD DAC2A", and a requirement copies
# the search sentence's "OECD DAC2a".
issues = check_evidence(
requirement(source=stated),
evidence_from_source(source(provenance_ref=None, provenance_label=label)),
)

assert errors(issues) == []


def test_a_stated_source_matches_a_provenance_label_across_accented_and_plain_spellings(
requirement: RequirementFactory, source: SourceFactory
) -> None:
Expand Down
62 changes: 54 additions & 8 deletions services/harness/tests/planner/test_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -1804,8 +1804,14 @@ def test_propose_plan_reads_the_dataset_of_every_pin_it_sums(
stored = load_plan(ctx.state)
assert stored is not None
aid = stored.requirement("aid")
assert isinstance(aid, DatasetRequirement)
assert aid.source_spec_id == SPEC_GHANA_AID
assert isinstance(aid, DatasetRequirement) and aid.source_spec_id is not None
if held_as == "shown":
assert aid.source_spec_id == SPEC_GHANA_AID
else:
# A found source's identity is on record, so the pin moves to its copy
# over the requirement's years.
copy = windowed_pins(ctx.state).get(aid.source_spec_id)
assert copy is not None and copy.windowed_from == SPEC_GHANA_AID


# The datasets `_sum_of_two_sources` adds by default, as a `combination_unasked`
Expand Down Expand Up @@ -8903,22 +8909,39 @@ def test_a_pin_a_source_choice_offered_is_judged_as_the_model_wrote_it(
_WHOLE, _WHOLE_IDENTITY = windowed_identity(_ON_EVERY_YEAR, open_window(_EVERY_YEAR_FROM))


@pytest.mark.parametrize("on_record", [True, False], ids=["on-record", "not-on-record"])
@pytest.mark.parametrize(
"held", ["searched", "discovered", "nowhere"], ids=["on-record", "in-the-log", "not-on-record"]
)
def test_a_pin_offered_for_a_copy_s_whole_history_reads_the_origin_s_stated_years(
on_record: bool,
held: str,
) -> None:
"""Discovery did not find the pin of 'aid_history', 'aid''s copy, and offers
`_WHOLE`, found over every year the dataset holds. The model reads the
choice for 'aid' and pins it there. With its identity on record, 'aid'
choice for 'aid' and pins it there. With its identity on record, from a
search in the conversation or as the source discovery logged, 'aid'
retrieves it from its stated 2015 and the copy from 1978. With none, the
years it holds are unknown, so the pin is refused for 'aid' and nothing
reads the whole history in its place."""
on_record = held != "nowhere"
lookup = {_STALE: RecordedIdentity(identity=_STALE_IDENTITY)}
if on_record:
if held == "searched":
lookup[_WHOLE] = RecordedIdentity(identity=_WHOLE_IDENTITY)
token = current_recorded_identity.set(lookup.get)
try:
ctx = _FakeToolContext()
if held == "discovered":
found = DiscoveryRecord(
revision="earlier",
tool="search",
requirement_id="aid_history",
requirement_key="earlier",
status=DiscoveryStatus.DEFINITE,
spec_ids=(_WHOLE,),
)
whole = SelectedSource(
spec_id=_WHOLE, sentence="gross aid", has_data=True, identity=_WHOLE_IDENTITY
)
save_log(ctx.state, load_log(ctx.state).with_record(found, (whole,)))
for spec_id in (_WHOLE, _STALE):
save_full_history(ctx.state, spec_id, FullHistory(first_year=_EVERY_YEAR_FROM, rows=35))
answered = [AskAnswer(ask=0, outputs=("aid",)), AskAnswer(ask=1, outputs=("top",))]
Expand Down Expand Up @@ -9072,17 +9095,40 @@ def _tools_reading(surface: ToolSurface) -> dict[str, FunctionTool]:
return {tool.name: tool for tool in build_planner_tools(surface=surface)}


@pytest.mark.usefixtures("aid_on_record")
@pytest.fixture(params=["searched", "discovered"])
def aid_held(request: pytest.FixtureRequest) -> str:
"""Where the 2020-2024 education-aid identity is on record: a search in the
conversation (`aid_on_record`), or discovery's log alone, which the test writes."""
if request.param == "searched":
request.getfixturevalue("aid_on_record")
return request.param


async def test_a_second_proposal_reads_no_first_year_the_turn_already_holds(
pinning_decision: Callable[..., Decision],
requirement: Callable[..., DatasetRequirement],
aid_held: str,
) -> None:
"""The first proposal reads the recorded dataset over no window once; an
amendment of the turn that still rolls from its start is pinned from the year
held."""
held. A dataset only discovery found, such as a referral's candidate, is on
record in its log and read the same way."""
reads = _DefinitionReads()
propose = _tools_reading(reads)["propose_plan"].func
ctx = _FakeToolContext()
if aid_held == "discovered":
found = DiscoveryRecord(
revision="earlier",
tool="search",
requirement_id="aid",
requirement_key="earlier",
status=DiscoveryStatus.DEFINITE,
spec_ids=(_AID_2020_2024_SPEC,),
)
aid = SelectedSource(
spec_id=_AID_2020_2024_SPEC, sentence="aid", has_data=True, identity=_AID_2020_2024
)
save_log(ctx.state, load_log(ctx.state).with_record(found, (aid,)))
dataset = requirement(
source_spec_id=_AID_2020_2024_SPEC,
period=Period(start=2020, end=2024),
Expand Down
18 changes: 15 additions & 3 deletions services/harness/tests/translator/test_discovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -790,6 +790,7 @@ def _matching_window() -> dict[str, Any]:
return {"kind": "exact", "has_data": True, "window": {"start_year": 2015, "end_year": 2024}}


@pytest.mark.parametrize("stated_source", [None, "OECD DAC2a"], ids=["no_source", "its_source"])
@pytest.mark.parametrize("earlier_turn", [False, True], ids=["this_turn", "an_earlier_turn"])
async def test_a_pin_of_a_shown_dataset_builds_the_source_with_no_search(
surface: McpSurface,
Expand All @@ -800,20 +801,31 @@ async def test_a_pin_of_a_shown_dataset_builds_the_source_with_no_search(
derived_requirement: DerivedRequirementFactory,
context_holding: ContextHolding,
earlier_turn: bool,
stated_source: str | None,
) -> None:
# The pin names a dataset the planner was shown (`PIN_SOURCES_STATE_KEY`)
# with no blocking warning, and the block's own window agrees with the
# requirement's period: the source comes straight from that block, with
# no `search` call at all. An earlier turn's dataset is shown with no
# block, and the pin reads the one that turn stored.
pinned = requirement(source_spec_id=SPEC_KENYA)
# block, and the pin reads the one that turn stored. A stated source the
# block's provenance names is confirmed by it, as "OECD DAC2a" is by
# Search's label "OECD DAC2A" for a search that named no source.
pinned = requirement(source_spec_id=SPEC_KENYA, source=stated_source)
context = context_holding(
draft(
decisions=[pinning_decision(SPEC_KENYA)],
evidence_requirements=[pinned, derived_requirement()],
)
)
block = _spec(SPEC_KENYA, "debt service paid by Kenya", coverage=_matching_window())
block = _spec(
SPEC_KENYA,
"debt service paid by Kenya",
coverage=_matching_window(),
facets={
"unit": {"basis": "declared_default", "ref": "usd", "label": "US dollars"},
"provenance": {"ref": "provenance/OECDDAC2A", "label": "OECD DAC2A"},
},
)
if earlier_turn:
sources = pin_sources(context.state)
sources[SPEC_KENYA] = PinSource(settled=True, earlier_turn=True)
Expand Down
Loading