From f0971cc17ef6b740a738a6d2b85c77854c9885f3 Mon Sep 17 00:00:00 2001 From: Radu Raicea Date: Fri, 20 Nov 2026 10:38:17 -0500 Subject: [PATCH 1/3] feat(llma): support offline evaluation updates and deletes --- .sampo/changesets/virtuous-shaman-kalma.md | 5 + .../example-offline-evaluations/README.md | 36 ++- posthog/ai/evaluations/__init__.py | 3 + posthog/ai/evaluations/_client.py | 170 ++++++++++++++ posthog/ai/evaluations/_mutations.py | 96 ++++++++ posthog/ai/evaluations/_serialization.py | 14 +- posthog/ai/evaluations/_transport.py | 14 +- posthog/test/ai/test_evaluations_mutations.py | 217 ++++++++++++++++++ references/public_api_snapshot.txt | 29 +++ 9 files changed, 576 insertions(+), 8 deletions(-) create mode 100644 .sampo/changesets/virtuous-shaman-kalma.md create mode 100644 posthog/ai/evaluations/_mutations.py create mode 100644 posthog/test/ai/test_evaluations_mutations.py diff --git a/.sampo/changesets/virtuous-shaman-kalma.md b/.sampo/changesets/virtuous-shaman-kalma.md new file mode 100644 index 000000000..35d508ea0 --- /dev/null +++ b/.sampo/changesets/virtuous-shaman-kalma.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: minor +--- + +Update and delete offline experiment items, reported scores, and payloads from sync and async clients using the current revision. diff --git a/examples/example-offline-evaluations/README.md b/examples/example-offline-evaluations/README.md index 188b2e22e..943607279 100644 --- a/examples/example-offline-evaluations/README.md +++ b/examples/example-offline-evaluations/README.md @@ -17,6 +17,9 @@ Offline evaluations must be enabled for your PostHog project. Use the project's | Create/update/archive/restore datasets and items | Personal API key with `dataset:write` | | Read experiment tags | Personal API key with `evaluation:read` | | Replace experiment tags | Personal API key with `evaluation:write` | +| Read an item or result revision | Personal API key with `evaluation:read`. Result reads also require `llm_analytics:read` and access to the scorer. | +| Update an item or update/delete its payload | Personal API key with `evaluation:write` and evaluation editor access | +| Delete an item, change/delete a result or its payload | Personal API key with `evaluation:write`, evaluation editor access, `llm_analytics:read`, and access to affected scorers and composite inputs | A personal key can cover both scorer management and uploads. Project secret keys can upload using a known scorer-version UUID without scorer read access. All operations use the explicit project/environment; scorer names are not unique. @@ -80,7 +83,7 @@ with OfflineEvaluations( Use one `EvaluationItem` for the application's execution and reuse it across scorer results. `upload_results([EvaluationResult(...), ...])` uploads a finite sequence, eagerly and sequentially. It splits at 1,000 items/results or 5 MiB of UTF-8 JSON per request, whichever comes first. Item payloads are limited to 1 MiB; result payloads to 256 KiB. Payload JSON supports at most 32 nested containers. The SDK raises for invalid values and oversized payloads without truncating them. Memory use scales with the sequence you supply, so submit caller-sized batches for large datasets. -Set `model_version` on each item to compare executions from different models in one experiment. It identifies the application's model and overrides the default supplied to `create_experiment(model_version=...)`. Omit it or pass `None` to inherit that default on the server; without either value, the model remains unknown. Explicit model identifiers must be nonempty strings of at most 255 characters and are preserved without trimming. The declared model is immutable item content: changing, adding, or removing it after acceptance causes a `content_conflict`, even if the effective model is unchanged. Create a new item for a different model execution, and preserve the original declaration for retries. +Set `model_version` on each item to compare executions from different models in one experiment. It identifies the application's model and overrides the default supplied to `create_experiment(model_version=...)`. Omit it or pass `None` to inherit that default on the server; without either value, the model remains unknown. Explicit model identifiers must be nonempty strings of at most 255 characters and are preserved without trimming. Re-uploading a retained item with a different declaration raises `content_conflict`. Use `update_item` to correct a recorded model, and create a new item for a different execution. Keep the original declaration for upload retries. `status="ok"` means the evaluator ran successfully. Its boolean, number, or category-list `value` is uploaded with its original polarity. Numeric scores use finite double-precision (binary64) values; integer inputs are rejected if conversion would change their value. Use `error`, `skipped`, or `not_applicable` without a value for other outcomes. `error_code` and a non-null `error_message` are valid only with `status="error"`. @@ -425,6 +428,37 @@ Reading and writing experiment tags require the personal-key scopes listed above The async client exposes the same API: await scorer methods and `experiment.get_tags()` / `experiment.set_tags(...)`. +## Correcting or deleting accepted data + +Use the item and result IDs from the upload receipt. You can make corrections while an experiment is uploading or after it completes or fails. With `AsyncOfflineEvaluations`, await each read, update, and delete. + +```python +experiment = evaluations.resume_experiment(experiment_id) +item_id = receipt.items[0].id +result_id = receipt.results[0].id + +revision = experiment.get_result_revision(result_id) +revision = experiment.update_result(result_id, revision=revision, status="ok", value=False) +experiment.update_result_payload(result_id, revision=revision, payload={"reasoning": "Reviewed by hand"}) + +revision = experiment.get_item_revision(item_id) +revision = experiment.update_item(item_id, revision=revision, model_version="v2") +experiment.update_item_payload(item_id, revision=revision, payload={"expected_output": None}) +``` + +Each update returns the server's current revision, including when a score already has the requested value. A payload shares its owner's revision. Changes to a result or its payload can also advance the item revision. A stale write raises `EvaluationAPIError` with HTTP 409 and code `revision_conflict`. After a conflict or lost response, inspect the current data before deciding whether to write again. The SDK sends each mutation once. + +Payload updates replace only the supplied top-level fields. Use `None` to store JSON null. Edits keep the original 30-day retention deadline. Editing an item's input, output, or expected output can mark existing scores stale. To correct a composite, edit its reported inputs and the server recomputes dependent scores. + +| Method | What it permanently removes | +| --- | --- | +| `delete_item(item_id, revision=...)` | The item, its results, and their payloads | +| `delete_result(result_id, revision=...)` | The result, its payload, and dependent composites | +| `delete_item_payload(item_id, revision=...)` | The stored item payload | +| `delete_result_payload(result_id, revision=...)` | The stored result payload | + +Deletes return `None` on acknowledgment. A deleted or expired payload cannot be restored on that record. Re-uploading a deleted item or result can recreate it while the experiment is open. A recreated reported result counts toward usage again. + ## Receipts, errors, and recovery Upload receipts contain persisted item/result UUIDs, original acceptance times, and `created` flags. `created=False` acknowledges an exact duplicate. A bulk upload returns `chunks`; each chunk commits independently. A malformed or incomplete acknowledgment raises rather than reporting success. diff --git a/posthog/ai/evaluations/__init__.py b/posthog/ai/evaluations/__init__.py index aa8c3e227..725fa7120 100644 --- a/posthog/ai/evaluations/__init__.py +++ b/posthog/ai/evaluations/__init__.py @@ -28,6 +28,7 @@ OfflineEvaluations, ) from ._errors import EvaluationAPIError +from ._mutations import ItemUpdate, ResultUpdate from ._scorer_types import ( BooleanScorerConfig, CategoricalPassingRule, @@ -93,6 +94,7 @@ "ExperimentOptions", "ExperimentReceipt", "ItemReceipt", + "ItemUpdate", "JSONValue", "NumericPassingRule", "NumericScorerConfig", @@ -100,6 +102,7 @@ "ResultOptions", "ResultReceipt", "ResultStatus", + "ResultUpdate", "RunSource", "ScoreValue", "Scorer", diff --git a/posthog/ai/evaluations/_client.py b/posthog/ai/evaluations/_client.py index 55c92c98e..49456e28b 100644 --- a/posthog/ai/evaluations/_client.py +++ b/posthog/ai/evaluations/_client.py @@ -14,14 +14,28 @@ from ._async_datasets import AsyncDatasets from ._datasets import Datasets from ._errors import EvaluationAPIError +from ._mutations import ( + ItemUpdate, + ResultUpdate, + _item_changes, + _item_path, + _item_payload_changes, + _patch_body, + _response_revision, + _result_changes, + _result_path, + _result_payload_changes, +) from ._scorers import AsyncScorers, Scorers from ._serialization import ( + JSONValue, _MAX_ITEMS, _MAX_REQUEST_BYTES, _MAX_RESULTS, _count, _encode, _identifier, + _integer, _timestamp, _uuid, ) @@ -282,6 +296,81 @@ def set_tags(self, tags: Sequence[str]) -> tuple[str, ...]: ) return _experiment_tags(data, self.id) + def get_item_revision(self, item_id: str | UUID) -> int: + """Read the current revision shared by an item and its payload.""" + path = self._path + _item_path(item_id) + with self._lock: + data = self._transport.request("GET", path) + return _response_revision(data, identity=item_id) + + def get_result_revision(self, result_id: str | UUID) -> int: + """Read the current result revision from its payload endpoint.""" + path = self._path + _result_path(result_id) + "payload/" + with self._lock: + data = self._transport.request("GET", path) + return _response_revision(data, identity=result_id) + + def _patch(self, path: str, revision: int, changes: dict[str, Any]) -> int: + body = _patch_body(revision, changes) + with self._lock: + data = self._transport.request( + "PATCH", self._path + path, body=body, retry_safe=False + ) + return _response_revision(data, minimum=revision) + + def _delete(self, path: str, revision: int) -> None: + params: dict[str, str | int] = {"revision": _integer(revision, minimum=0)} + with self._lock: + self._transport.request( + "DELETE", self._path + path, params=params, retry_safe=False + ) + + def update_item( + self, item_id: str | UUID, *, revision: int, **fields: Unpack[ItemUpdate] + ) -> int: + """Update item identifiers at the observed revision and return the current revision.""" + return self._patch(_item_path(item_id), revision, _item_changes(fields)) + + def update_result( + self, result_id: str | UUID, *, revision: int, **fields: Unpack[ResultUpdate] + ) -> int: + """Correct a reported score and return its revision. Dependent composites are recalculated.""" + return self._patch(_result_path(result_id), revision, _result_changes(fields)) + + def update_item_payload( + self, item_id: str | UUID, *, revision: int, payload: dict[str, JSONValue] + ) -> int: + """Patch input, output, expected_output, or metadata and return the item revision.""" + return self._patch( + _item_path(item_id) + "payload/", revision, _item_payload_changes(payload) + ) + + def update_result_payload( + self, result_id: str | UUID, *, revision: int, payload: dict[str, JSONValue] + ) -> int: + """Patch reasoning, error_message, or metadata and return the result revision.""" + return self._patch( + _result_path(result_id) + "payload/", + revision, + _result_payload_changes(payload), + ) + + def delete_item(self, item_id: str | UUID, *, revision: int) -> None: + """Permanently delete an item and all its results and payloads.""" + self._delete(_item_path(item_id), revision) + + def delete_result(self, result_id: str | UUID, *, revision: int) -> None: + """Permanently delete a result, its payload, and dependent composites.""" + self._delete(_result_path(result_id), revision) + + def delete_item_payload(self, item_id: str | UUID, *, revision: int) -> None: + """Permanently remove the stored payload while retaining the item and scores.""" + self._delete(_item_path(item_id) + "payload/", revision) + + def delete_result_payload(self, result_id: str | UUID, *, revision: int) -> None: + """Permanently remove the stored payload while retaining its result.""" + self._delete(_result_path(result_id) + "payload/", revision) + def upload_result( self, *, @@ -412,6 +501,85 @@ async def set_tags(self, tags: Sequence[str]) -> tuple[str, ...]: ) return _experiment_tags(data, self.id) + async def get_item_revision(self, item_id: str | UUID) -> int: + """Read the current revision shared by an item and its payload.""" + path = self._path + _item_path(item_id) + async with self._lock: + data = await self._transport.request("GET", path) + return _response_revision(data, identity=item_id) + + async def get_result_revision(self, result_id: str | UUID) -> int: + """Read the current result revision from its payload endpoint.""" + path = self._path + _result_path(result_id) + "payload/" + async with self._lock: + data = await self._transport.request("GET", path) + return _response_revision(data, identity=result_id) + + async def _patch(self, path: str, revision: int, changes: dict[str, Any]) -> int: + body = _patch_body(revision, changes) + async with self._lock: + data = await self._transport.request( + "PATCH", self._path + path, body=body, retry_safe=False + ) + return _response_revision(data, minimum=revision) + + async def _delete(self, path: str, revision: int) -> None: + params: dict[str, str | int] = {"revision": _integer(revision, minimum=0)} + async with self._lock: + await self._transport.request( + "DELETE", self._path + path, params=params, retry_safe=False + ) + + async def update_item( + self, item_id: str | UUID, *, revision: int, **fields: Unpack[ItemUpdate] + ) -> int: + """Update item identifiers at the observed revision and return the current revision.""" + return await self._patch(_item_path(item_id), revision, _item_changes(fields)) + + async def update_result( + self, result_id: str | UUID, *, revision: int, **fields: Unpack[ResultUpdate] + ) -> int: + """Correct a reported score and return its revision. Dependent composites are recalculated.""" + return await self._patch( + _result_path(result_id), revision, _result_changes(fields) + ) + + async def update_item_payload( + self, item_id: str | UUID, *, revision: int, payload: dict[str, JSONValue] + ) -> int: + """Patch input, output, expected_output, or metadata and return the item revision.""" + return await self._patch( + _item_path(item_id) + "payload/", revision, _item_payload_changes(payload) + ) + + async def update_result_payload( + self, result_id: str | UUID, *, revision: int, payload: dict[str, JSONValue] + ) -> int: + """Patch reasoning, error_message, or metadata and return the result revision.""" + return await self._patch( + _result_path(result_id) + "payload/", + revision, + _result_payload_changes(payload), + ) + + async def delete_item(self, item_id: str | UUID, *, revision: int) -> None: + """Permanently delete an item and all its results and payloads.""" + await self._delete(_item_path(item_id), revision) + + async def delete_result(self, result_id: str | UUID, *, revision: int) -> None: + """Permanently delete a result, its payload, and dependent composites.""" + await self._delete(_result_path(result_id), revision) + + async def delete_item_payload(self, item_id: str | UUID, *, revision: int) -> None: + """Permanently remove the stored payload while retaining the item and scores.""" + await self._delete(_item_path(item_id) + "payload/", revision) + + async def delete_result_payload( + self, result_id: str | UUID, *, revision: int + ) -> None: + """Permanently remove the stored payload while retaining its result.""" + await self._delete(_result_path(result_id) + "payload/", revision) + async def upload_result( self, *, @@ -480,6 +648,8 @@ class OfflineEvaluations: ``secret_key`` accepts personal or project secret keys with offline_evaluation_ingestion:write. Scorer reads/writes require a personal key with llm_analytics:read/write; datasets require dataset:read/write. + Editing accepted data requires an evaluation editor's personal key with + evaluation:write. Result operations and item deletion also need scorer access. Each operation requires its corresponding product feature to be enabled. ``host`` defaults to the US app API; known ingestion hosts are mapped to their app hosts. Custom hosts are preserved. Requests use a 15 second diff --git a/posthog/ai/evaluations/_mutations.py b/posthog/ai/evaluations/_mutations.py new file mode 100644 index 000000000..5d64de580 --- /dev/null +++ b/posthog/ai/evaluations/_mutations.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from typing import Any, Mapping +from uuid import UUID + +from typing_extensions import TypedDict + +from ._errors import EvaluationAPIError +from ._serialization import ( + JSONValue, + _MAX_ITEM_BYTES, + _MAX_RESULT_BYTES, + _checked_payload, + _encode, + _identifier, + _integer, + _score, + _uuid, +) +from ._types import ResultStatus, ScoreValue, _validate_result_status + + +class ItemUpdate(TypedDict, total=False): + """Editable experiment item fields. An explicit None clears the field.""" + + case_key: str | None + trial: str | None + model_version: str | None + application_trace_id: str | None + + +class ResultUpdate(TypedDict, total=False): + """Fields for correcting the outcome of a reported result.""" + + status: ResultStatus + value: ScoreValue | None + error_code: str | None + + +def _item_path(item_id: str | UUID) -> str: + return f"items/{_uuid(item_id)}/" + + +def _result_path(result_id: str | UUID) -> str: + return f"results/{_uuid(result_id)}/" + + +def _item_changes(fields: ItemUpdate) -> dict[str, Any]: + if fields.keys() - ItemUpdate.__annotations__: + raise TypeError("Unsupported item update field.") + return {key: _identifier(value, key) for key, value in fields.items()} + + +def _result_changes(fields: ResultUpdate) -> dict[str, Any]: + if fields.keys() - ResultUpdate.__annotations__: + raise TypeError("Unsupported result update field.") + changes: dict[str, Any] = dict(fields) + if "status" in fields: + _validate_result_status( + fields["status"], fields.get("value"), fields.get("error_code") + ) + if (value := fields.get("value")) is not None: + changes["value"] = _score(value) + if "error_code" in fields: + changes["error_code"] = _identifier(fields["error_code"], "error_code", 128) + return changes + + +def _item_payload_changes(payload: dict[str, JSONValue]) -> dict[str, Any]: + fields = ("input", "output", "expected_output", "metadata") + return {"payload": _checked_payload(payload, fields, _MAX_ITEM_BYTES)} + + +def _result_payload_changes(payload: dict[str, JSONValue]) -> dict[str, Any]: + fields = ("reasoning", "error_message", "metadata") + return {"payload": _checked_payload(payload, fields, _MAX_RESULT_BYTES)} + + +def _patch_body(revision: int, changes: Mapping[str, Any]) -> bytes: + if not changes: + raise ValueError("Supply at least one field to update.") + return _encode({"revision": _integer(revision, minimum=0), **changes}) + + +def _response_revision( + data: dict[str, Any], *, identity: str | UUID | None = None, minimum: int = 0 +) -> int: + try: + if identity is not None and _uuid(data["id"]) != _uuid(identity): + raise ValueError("Mismatched resource ID.") + return _integer(data["revision"], minimum=minimum) + except (KeyError, TypeError, ValueError): + raise EvaluationAPIError( + code="invalid_response", + detail="Expected the current revision of the requested resource.", + ) from None diff --git a/posthog/ai/evaluations/_serialization.py b/posthog/ai/evaluations/_serialization.py index c651bad11..e9002ed91 100644 --- a/posthog/ai/evaluations/_serialization.py +++ b/posthog/ai/evaluations/_serialization.py @@ -3,7 +3,7 @@ import json import math from datetime import datetime, timezone -from typing import Any, Callable, Iterable, TypeVar, Union, cast +from typing import Any, Callable, Collection, Iterable, TypeVar, Union, cast from urllib.parse import parse_qs, urlsplit from uuid import UUID @@ -151,13 +151,17 @@ def _payload( if not isinstance(explicit, _Unset): if supplied: raise ValueError("Supply payload or individual payload fields, not both.") - if not isinstance(explicit, dict) or set(explicit) - set(fields): - raise ValueError( - "payload must be an object containing supported fields only." - ) supplied = explicit elif not supplied: return _UNSET + return _checked_payload(supplied, fields.keys(), limit) + + +def _checked_payload( + supplied: dict[str, JSONValue], allowed: Collection[str], limit: int +) -> dict[str, JSONValue]: + if not isinstance(supplied, dict) or supplied.keys() - allowed: + raise ValueError("payload must be an object containing supported fields only.") _validate_payload_fields(supplied) _validate_json(supplied) encoded = _encode(supplied) diff --git a/posthog/ai/evaluations/_transport.py b/posthog/ai/evaluations/_transport.py index 1feab76f0..77f182339 100644 --- a/posthog/ai/evaluations/_transport.py +++ b/posthog/ai/evaluations/_transport.py @@ -124,10 +124,18 @@ def _retry_delay( def _parse_response( - status: int, payload: Any, retry_after: str | None + method: str, status: int, payload: Any, retry_after: str | None ) -> dict[str, Any]: if 200 <= status < 300: - if isinstance(payload, dict): + if method == "DELETE": + if status == 204: + return {} + raise EvaluationAPIError( + status=status, + code="invalid_response", + detail="Expected HTTP 204 to acknowledge deletion.", + ) + if status != 204 and isinstance(payload, dict): return payload raise EvaluationAPIError( status=status, @@ -214,6 +222,7 @@ def request( except ValueError: payload = None return _parse_response( + method, response.status_code, payload, response.headers.get("Retry-After"), @@ -292,6 +301,7 @@ async def request( except ValueError: payload = None return _parse_response( + method, response.status_code, payload, response.headers.get("Retry-After"), diff --git a/posthog/test/ai/test_evaluations_mutations.py b/posthog/test/ai/test_evaluations_mutations.py new file mode 100644 index 000000000..7c1a0a9d0 --- /dev/null +++ b/posthog/test/ai/test_evaluations_mutations.py @@ -0,0 +1,217 @@ +import json +from unittest.mock import patch +from uuid import UUID + +import httpx +import pytest +import requests + +from posthog.ai.evaluations import ( + AsyncOfflineEvaluations, + EvaluationAPIError, + OfflineEvaluations, +) + +from .test_evaluations import BASE, EXPERIMENT_ID, invoke + + +RESOURCE_ID = UUID("e717c026-18ae-4640-a33e-109c19c97328") +EXPERIMENT_URL = f"https://us.posthog.com{BASE}{EXPERIMENT_ID}/" + + +@pytest.fixture(params=[False, True], ids=["sync", "async"]) +def api(request): + """Exercise both public clients down to the prepared HTTP request.""" + asynchronous = request.param + client = (AsyncOfflineEvaluations if asynchronous else OfflineEvaluations)( + project_id=123, secret_key="test-secret" + ) + http = client._transport._client if asynchronous else client._transport._session + + def respond(status, data=None): + body = b"" if data is None else json.dumps(data).encode() + if asynchronous: + return httpx.Response(status, content=body) + response = requests.Response() + response.status_code = status + response._content = body + return response + + with patch.object(http, "send") as send: + try: + yield client.resume_experiment(EXPERIMENT_ID), send, respond + finally: + invoke(client.aclose if asynchronous else client.close) + + +@pytest.mark.parametrize( + "resource,collection,revision_suffix,fields,payload", + [ + ( + "item", + "items", + "", + { + "case_key": None, + "trial": "2", + "model_version": "v2", + "application_trace_id": "trace", + }, + {"output": None}, + ), + ( + "result", + "results", + "payload/", + {"status": "ok", "value": False}, + {"metadata": {"reviewed": True}, "reasoning": None}, + ), + ], +) +def test_read_update_and_delete_with_revisions( + api, resource, collection, revision_suffix, fields, payload +): + experiment, send, respond = api + # An unchanged score keeps its revision. Item updates advance theirs. + after_update = 0 if resource == "result" else 1 + after_payload = after_update + 1 + send.side_effect = [ + respond(200, {"id": str(RESOURCE_ID), "revision": 0}), + respond(200, {"revision": after_update}), + respond(200, {"revision": after_payload}), + respond(204), + respond(200, {"id": str(RESOURCE_ID), "revision": after_payload + 1}), + respond(204), + ] + read = getattr(experiment, f"get_{resource}_revision") + revision = invoke(read, RESOURCE_ID) + assert revision == 0 + revision = invoke( + getattr(experiment, f"update_{resource}"), + RESOURCE_ID, + revision=revision, + **fields, + ) + assert revision == after_update + revision = invoke( + getattr(experiment, f"update_{resource}_payload"), + RESOURCE_ID, + revision=revision, + payload=payload, + ) + assert revision == after_payload + assert ( + invoke( + getattr(experiment, f"delete_{resource}_payload"), + RESOURCE_ID, + revision=revision, + ) + is None + ) + revision = invoke(read, RESOURCE_ID) + assert ( + invoke( + getattr(experiment, f"delete_{resource}"), RESOURCE_ID, revision=revision + ) + is None + ) + + url = f"{EXPERIMENT_URL}{collection}/{RESOURCE_ID}/" + requests_sent = [call.args[0] for call in send.call_args_list] + assert [(req.method, str(req.url)) for req in requests_sent] == [ + ("GET", url + revision_suffix), + ("PATCH", url), + ("PATCH", url + "payload/"), + ("DELETE", url + f"payload/?revision={after_payload}"), + ("GET", url + revision_suffix), + ("DELETE", url + f"?revision={after_payload + 1}"), + ] + bodies = [ + req.content if isinstance(req, httpx.Request) else req.body + for req in requests_sent + ] + assert [json.loads(body) if body else None for body in bodies] == [ + None, + {"revision": 0, **fields}, + {"revision": after_update, "payload": payload}, + None, + None, + None, + ] + + +@pytest.mark.parametrize("method", ["update_result", "delete_result"]) +@pytest.mark.parametrize( + "status,code,persistence", + [(409, "revision_conflict", "rejected"), (503, "unavailable", "unknown")], +) +def test_mutations_surface_errors_without_replaying( + api, method, status, code, persistence +): + experiment, send, respond = api + send.side_effect = [respond(status, {"code": code})] + fields = {"error_code": "timeout"} if method == "update_result" else {} + with pytest.raises(EvaluationAPIError) as caught: + invoke(getattr(experiment, method), RESOURCE_ID, revision=1, **fields) + assert (caught.value.status, caught.value.code, caught.value.persistence) == ( + status, + code, + persistence, + ) + send.assert_called_once() + + +@pytest.mark.parametrize( + "method,fields,status,response", + [ + ("get_item_revision", {}, 200, {"id": EXPERIMENT_ID, "revision": 2}), + ("get_result_revision", {}, 200, {"id": str(RESOURCE_ID)}), + ("update_result", {"revision": 0, "value": 0}, 204, None), + ("update_item", {"revision": 0, "trial": None}, 200, {"revision": False}), + ("update_item", {"revision": 3, "trial": None}, 200, {"revision": 2}), + ("delete_item", {"revision": 0}, 202, {}), + ], +) +def test_malformed_acknowledgments_have_unknown_outcome( + api, method, fields, status, response +): + experiment, send, respond = api + send.side_effect = [respond(status, response)] + with pytest.raises(EvaluationAPIError) as caught: + invoke(getattr(experiment, method), RESOURCE_ID, **fields) + assert caught.value.code == "invalid_response" + assert caught.value.persistence == "unknown" + send.assert_called_once() + + +@pytest.mark.parametrize( + "method,identity,fields", + [ + ("update_item", RESOURCE_ID, {"revision": 0}), + ("update_result", RESOURCE_ID, {"revision": 1, "value": 2**53 + 1}), + ( + "update_result", + RESOURCE_ID, + {"revision": 1, "status": "error", "value": False}, + ), + ( + "update_result_payload", + RESOURCE_ID, + {"revision": 1, "payload": {"error_message": 2}}, + ), + ( + "update_item_payload", + RESOURCE_ID, + {"revision": 1, "payload": {"reasoning": "unexpected"}}, + ), + ("update_item", RESOURCE_ID, {"revision": 0, "dataset_item_version_id": None}), + ("delete_item", RESOURCE_ID, {"revision": False}), + ("delete_result_payload", RESOURCE_ID, {"revision": -1}), + ("delete_result", "../items", {"revision": 0}), + ], +) +def test_invalid_mutations_send_nothing(api, method, identity, fields): + experiment, send, _ = api + with pytest.raises((ValueError, TypeError)): + invoke(getattr(experiment, method), identity, **fields) + send.assert_not_called() diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index c9e0b430e..d6063b7b2 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -507,6 +507,10 @@ attribute posthog.ai.evaluations.ExperimentReceipt.status: Literal['uploading', attribute posthog.ai.evaluations.ItemReceipt.accepted_at: datetime attribute posthog.ai.evaluations.ItemReceipt.created: bool attribute posthog.ai.evaluations.ItemReceipt.id: str +attribute posthog.ai.evaluations.ItemUpdate.application_trace_id: str | None +attribute posthog.ai.evaluations.ItemUpdate.case_key: str | None +attribute posthog.ai.evaluations.ItemUpdate.model_version: str | None +attribute posthog.ai.evaluations.ItemUpdate.trial: str | None attribute posthog.ai.evaluations.JSONValue = Union[None, bool, int, float, str, list['JSONValue'], dict[str, 'JSONValue']] attribute posthog.ai.evaluations.NumericPassingRule.operator: Literal['gte', 'lte'] attribute posthog.ai.evaluations.NumericPassingRule.threshold: float @@ -531,6 +535,9 @@ attribute posthog.ai.evaluations.ResultReceipt.item_id: str attribute posthog.ai.evaluations.ResultReceipt.scorer_version_id: str attribute posthog.ai.evaluations.ResultReceipt.status: ResultStatus | None = None attribute posthog.ai.evaluations.ResultStatus = Literal['ok', 'error', 'skipped', 'not_applicable'] +attribute posthog.ai.evaluations.ResultUpdate.error_code: str | None +attribute posthog.ai.evaluations.ResultUpdate.status: ResultStatus +attribute posthog.ai.evaluations.ResultUpdate.value: ScoreValue | None attribute posthog.ai.evaluations.RunSource = Literal['ci', 'local', 'scheduled'] attribute posthog.ai.evaluations.ScoreValue = bool | int | float | list[str] attribute posthog.ai.evaluations.Scorer.archived: bool @@ -1310,11 +1317,13 @@ class posthog.ai.evaluations.Experiment(transport: SyncTransport, project_id: in class posthog.ai.evaluations.ExperimentOptions class posthog.ai.evaluations.ExperimentReceipt(id: str, status: Literal['uploading', 'completed', 'failed'], created: bool, started_at: datetime, created_at: datetime, finished_at: datetime | None, expected_item_count: int | None, expected_result_count: int | None, accepted_item_count: int, accepted_result_count: int) class posthog.ai.evaluations.ItemReceipt(id: str, created: bool, accepted_at: datetime) +class posthog.ai.evaluations.ItemUpdate class posthog.ai.evaluations.NumericPassingRule class posthog.ai.evaluations.NumericScorerConfig class posthog.ai.evaluations.OfflineEvaluations(*, project_id: int, secret_key: str, host: str | None = None, timeout: float = 15, max_retries: int = 3) class posthog.ai.evaluations.ResultOptions class posthog.ai.evaluations.ResultReceipt(id: str, created: bool, accepted_at: datetime, item_id: str, scorer_version_id: str, status: ResultStatus | None = None, error_code: str | None = None) +class posthog.ai.evaluations.ResultUpdate class posthog.ai.evaluations.Scorer(id: str, name: str, description: str, kind: ScorerKind, archived: bool, current_version: int, current_version_id: str, config: _ConfigT, created_at: datetime, updated_at: datetime, created_by: dict[str, Any] | None, team: int, tags: tuple[str, ...] = (), calculation: ScorerCalculation = 'reported', composition: ScorerComposition | None = None, input_scorers: tuple[ScorerInput, ...] = (), dependent_composite_count: int = 0) class posthog.ai.evaluations.ScorerComposition class posthog.ai.evaluations.ScorerCompositionInput @@ -1706,9 +1715,19 @@ method posthog.ai.evaluations.AsyncDatasets.snapshot(dataset_id: str | UUID, *, method posthog.ai.evaluations.AsyncDatasets.update(dataset_id: str | UUID, *, name: str | _Unset = _UNSET, description: str | _Unset = _UNSET, metadata: dict[str, JSONValue] | _Unset = _UNSET, tags: Sequence[str] | _Unset = _UNSET) method posthog.ai.evaluations.AsyncDatasets.update_item(item_id: str | UUID, *, base_version: int, input: JSONValue | _Unset = _UNSET, expected_output: JSONValue | _Unset = _UNSET, metadata: dict[str, JSONValue] | _Unset = _UNSET) method posthog.ai.evaluations.AsyncExperiment.complete() +method posthog.ai.evaluations.AsyncExperiment.delete_item(item_id: str | UUID, *, revision: int) +method posthog.ai.evaluations.AsyncExperiment.delete_item_payload(item_id: str | UUID, *, revision: int) +method posthog.ai.evaluations.AsyncExperiment.delete_result(result_id: str | UUID, *, revision: int) +method posthog.ai.evaluations.AsyncExperiment.delete_result_payload(result_id: str | UUID, *, revision: int) method posthog.ai.evaluations.AsyncExperiment.fail() +method posthog.ai.evaluations.AsyncExperiment.get_item_revision(item_id: str | UUID) +method posthog.ai.evaluations.AsyncExperiment.get_result_revision(result_id: str | UUID) method posthog.ai.evaluations.AsyncExperiment.get_tags() method posthog.ai.evaluations.AsyncExperiment.set_tags(tags: Sequence[str]) +method posthog.ai.evaluations.AsyncExperiment.update_item(item_id: str | UUID, *, revision: int, **fields: Unpack[ItemUpdate]) +method posthog.ai.evaluations.AsyncExperiment.update_item_payload(item_id: str | UUID, *, revision: int, payload: dict[str, JSONValue]) +method posthog.ai.evaluations.AsyncExperiment.update_result(result_id: str | UUID, *, revision: int, **fields: Unpack[ResultUpdate]) +method posthog.ai.evaluations.AsyncExperiment.update_result_payload(result_id: str | UUID, *, revision: int, payload: dict[str, JSONValue]) method posthog.ai.evaluations.AsyncExperiment.upload_result(*, item: EvaluationItem | str | UUID, scorer_version_id: str | UUID, **options: Unpack[ResultOptions]) method posthog.ai.evaluations.AsyncExperiment.upload_results(results: Sequence[EvaluationResult]) method posthog.ai.evaluations.AsyncOfflineEvaluations.aclose() @@ -1744,9 +1763,19 @@ method posthog.ai.evaluations.EvaluationItem.to_dict() method posthog.ai.evaluations.EvaluationResult.from_dict(data: Mapping[str, Any], *, item: EvaluationItem | None = None) method posthog.ai.evaluations.EvaluationResult.to_dict() method posthog.ai.evaluations.Experiment.complete() +method posthog.ai.evaluations.Experiment.delete_item(item_id: str | UUID, *, revision: int) +method posthog.ai.evaluations.Experiment.delete_item_payload(item_id: str | UUID, *, revision: int) +method posthog.ai.evaluations.Experiment.delete_result(result_id: str | UUID, *, revision: int) +method posthog.ai.evaluations.Experiment.delete_result_payload(result_id: str | UUID, *, revision: int) method posthog.ai.evaluations.Experiment.fail() +method posthog.ai.evaluations.Experiment.get_item_revision(item_id: str | UUID) +method posthog.ai.evaluations.Experiment.get_result_revision(result_id: str | UUID) method posthog.ai.evaluations.Experiment.get_tags() method posthog.ai.evaluations.Experiment.set_tags(tags: Sequence[str]) +method posthog.ai.evaluations.Experiment.update_item(item_id: str | UUID, *, revision: int, **fields: Unpack[ItemUpdate]) +method posthog.ai.evaluations.Experiment.update_item_payload(item_id: str | UUID, *, revision: int, payload: dict[str, JSONValue]) +method posthog.ai.evaluations.Experiment.update_result(result_id: str | UUID, *, revision: int, **fields: Unpack[ResultUpdate]) +method posthog.ai.evaluations.Experiment.update_result_payload(result_id: str | UUID, *, revision: int, payload: dict[str, JSONValue]) method posthog.ai.evaluations.Experiment.upload_result(*, item: EvaluationItem | str | UUID, scorer_version_id: str | UUID, **options: Unpack[ResultOptions]) method posthog.ai.evaluations.Experiment.upload_results(results: Sequence[EvaluationResult]) method posthog.ai.evaluations.OfflineEvaluations.close() From f8313834711852f8cc8997e1889aaf96940f0f75 Mon Sep 17 00:00:00 2001 From: Radu Raicea Date: Fri, 20 Nov 2026 14:32:35 -0500 Subject: [PATCH 2/3] chore(llma): cover empty offline evaluation payload updates --- .../example-offline-evaluations/README.md | 4 ++++ posthog/test/ai/test_evaluations_mutations.py | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/examples/example-offline-evaluations/README.md b/examples/example-offline-evaluations/README.md index 943607279..faa7f73e0 100644 --- a/examples/example-offline-evaluations/README.md +++ b/examples/example-offline-evaluations/README.md @@ -448,8 +448,12 @@ experiment.update_item_payload(item_id, revision=revision, payload={"expected_ou Each update returns the server's current revision, including when a score already has the requested value. A payload shares its owner's revision. Changes to a result or its payload can also advance the item revision. A stale write raises `EvaluationAPIError` with HTTP 409 and code `revision_conflict`. After a conflict or lost response, inspect the current data before deciding whether to write again. The SDK sends each mutation once. +If an update acknowledgment returns a revision below the one you submitted, the SDK raises `EvaluationAPIError` with code `invalid_response` and `persistence="unknown"`. + Payload updates replace only the supplied top-level fields. Use `None` to store JSON null. Edits keep the original 30-day retention deadline. Editing an item's input, output, or expected output can mark existing scores stale. To correct a composite, edit its reported inputs and the server recomputes dependent scores. +Passing `payload={}` stores an empty first payload or preserves the fields of an existing payload. Either write advances the owner's revision. A deleted or expired payload raises HTTP 409 `payload_unavailable` even for `{}`. When you change a result's status from `error` to another status, the server removes any stored `error_message` from its payload. + | Method | What it permanently removes | | --- | --- | | `delete_item(item_id, revision=...)` | The item, its results, and their payloads | diff --git a/posthog/test/ai/test_evaluations_mutations.py b/posthog/test/ai/test_evaluations_mutations.py index 7c1a0a9d0..440e8fdc2 100644 --- a/posthog/test/ai/test_evaluations_mutations.py +++ b/posthog/test/ai/test_evaluations_mutations.py @@ -140,6 +140,26 @@ def test_read_update_and_delete_with_revisions( ] +@pytest.mark.parametrize("method", ["update_item_payload", "update_result_payload"]) +def test_empty_payload_update_is_sent(api, method): + experiment, send, respond = api + update = getattr(experiment, method) + send.side_effect = [ + respond(200, {"revision": 4}), + respond(409, {"code": "payload_unavailable"}), + ] + + assert invoke(update, RESOURCE_ID, revision=3, payload={}) == 4 + with pytest.raises(EvaluationAPIError) as caught: + invoke(update, RESOURCE_ID, revision=4, payload={}) + assert (caught.value.status, caught.value.code) == (409, "payload_unavailable") + + for call, revision in zip(send.call_args_list, [3, 4], strict=True): + request = call.args[0] + body = request.content if isinstance(request, httpx.Request) else request.body + assert json.loads(body) == {"revision": revision, "payload": {}} + + @pytest.mark.parametrize("method", ["update_result", "delete_result"]) @pytest.mark.parametrize( "status,code,persistence", From 71b24b3408f5b33d4a213f7826fa5924d4eff1be Mon Sep 17 00:00:00 2001 From: Radu Raicea Date: Fri, 20 Nov 2026 15:16:06 -0500 Subject: [PATCH 3/3] fix(llma): validate offline evaluation result updates --- .../example-offline-evaluations/README.md | 6 +- posthog/ai/evaluations/_mutations.py | 15 +++-- posthog/test/ai/test_evaluations_mutations.py | 60 +++++++++++++++++++ 3 files changed, 74 insertions(+), 7 deletions(-) diff --git a/examples/example-offline-evaluations/README.md b/examples/example-offline-evaluations/README.md index faa7f73e0..ced127c17 100644 --- a/examples/example-offline-evaluations/README.md +++ b/examples/example-offline-evaluations/README.md @@ -155,7 +155,7 @@ The SDK keeps all supplied results for a composite-bearing item together without HTTP success acknowledges persistence, not successful calculation. Input errors become `composite_input_error`; formula errors or invalid output values become `composite_formula_error`. Input statuses propagate in the order `error`, `skipped`, `not_applicable`. Check each receipt's `status` and `error_code`; no result-read permission is needed. For ordinary results on an older backend, `status=None` means the backend omitted the outcome. Composite acknowledgments require an outcome. -All accepted composite outcomes count toward `expected_result_count`, including errors and skips. Derived results add no uploaded-result billing usage. Omitting a composite leaves it unrequested. Exact retries return the stored outcome, including after completion; formulas are not recalculated on retry or read. Persist declarations with `to_dict` and restore them with `from_dict` for exact recovery. +All accepted composite outcomes count toward `expected_result_count`, including errors and skips. Derived results add no uploaded-result billing usage. Omitting a composite leaves it unrequested. An exact upload retry for a retained result returns its original outcome, even after an edit or experiment completion. The server does not recalculate formulas on retry or read. Persist declarations with `to_dict` and restore them with `from_dict` for exact recovery. Formulas support 1–16 reported inputs and at most 1,024 characters; aliases start with a lowercase letter and contain lowercase letters, digits, or underscores, up to 32 characters. Nested composites are unsupported. Use `number(flag)` for explicit boolean conversion and `"good" in categories` for categorical membership. Boolean inputs retain their raw meaning regardless of passing rules. Numeric composites support bounds and passing thresholds, but no non-null `step`. Formula syntax, input access, and output types are validated by the server; runtime values and bounds are checked during ingestion. @@ -461,7 +461,9 @@ Passing `payload={}` stores an empty first payload or preserves the fields of an | `delete_item_payload(item_id, revision=...)` | The stored item payload | | `delete_result_payload(result_id, revision=...)` | The stored result payload | -Deletes return `None` on acknowledgment. A deleted or expired payload cannot be restored on that record. Re-uploading a deleted item or result can recreate it while the experiment is open. A recreated reported result counts toward usage again. +Deletes return `None` on acknowledgment. Deleting a payload advances its owner's revision. Deleting a result or its payload also advances the parent item's revision. Before editing an affected record that still exists, read its new revision with `get_item_revision` or `get_result_revision`. + +A deleted or expired payload cannot be restored on that record. Re-uploading a deleted item or result can recreate it while the experiment is open. A recreated reported result counts toward usage again. ## Receipts, errors, and recovery diff --git a/posthog/ai/evaluations/_mutations.py b/posthog/ai/evaluations/_mutations.py index 5d64de580..0fc9f32cc 100644 --- a/posthog/ai/evaluations/_mutations.py +++ b/posthog/ai/evaluations/_mutations.py @@ -55,14 +55,19 @@ def _result_changes(fields: ResultUpdate) -> dict[str, Any]: if fields.keys() - ResultUpdate.__annotations__: raise TypeError("Unsupported result update field.") changes: dict[str, Any] = dict(fields) + value = fields.get("value") + error_code = fields.get("error_code") if "status" in fields: - _validate_result_status( - fields["status"], fields.get("value"), fields.get("error_code") - ) - if (value := fields.get("value")) is not None: + status = fields["status"] + _validate_result_status(status, value, error_code) + if status == "ok" and "value" in fields and value is None: + raise ValueError("An ok result requires a score.") + elif value is not None and error_code is not None: + raise ValueError("A result cannot contain both a score and an error_code.") + if value is not None: changes["value"] = _score(value) if "error_code" in fields: - changes["error_code"] = _identifier(fields["error_code"], "error_code", 128) + changes["error_code"] = _identifier(error_code, "error_code", 128) return changes diff --git a/posthog/test/ai/test_evaluations_mutations.py b/posthog/test/ai/test_evaluations_mutations.py index 440e8fdc2..b72ea4509 100644 --- a/posthog/test/ai/test_evaluations_mutations.py +++ b/posthog/test/ai/test_evaluations_mutations.py @@ -7,6 +7,7 @@ import requests from posthog.ai.evaluations import ( + AsyncExperiment, AsyncOfflineEvaluations, EvaluationAPIError, OfflineEvaluations, @@ -140,6 +141,41 @@ def test_read_update_and_delete_with_revisions( ] +def test_ok_update_can_omit_value(api): + experiment, send, respond = api + send.return_value = respond(200, {"revision": 0}) + + assert invoke(experiment.update_result, RESOURCE_ID, revision=0, status="ok") == 0 + send.assert_called_once() + request = send.call_args.args[0] + body = request.content if isinstance(request, httpx.Request) else request.body + assert json.loads(body) == {"revision": 0, "status": "ok"} + + +def test_retry_revision_read_then_delete_with_revision_zero(api): + experiment, send, respond = api + send.side_effect = [ + respond(503, {"code": "unavailable"}), + respond(200, {"id": str(RESOURCE_ID), "revision": 0}), + respond(204), + ] + with ( + patch("posthog.ai.evaluations._transport.time.sleep"), + patch("posthog.ai.evaluations._transport.asyncio.sleep"), + ): + revision = invoke(experiment.get_item_revision, RESOURCE_ID) + assert revision == 0 + assert invoke(experiment.delete_item, RESOURCE_ID, revision=revision) is None + + url = f"{EXPERIMENT_URL}items/{RESOURCE_ID}/" + requests_sent = [call.args[0] for call in send.call_args_list] + assert [(req.method, str(req.url)) for req in requests_sent] == [ + ("GET", url), + ("GET", url), + ("DELETE", url + "?revision=0"), + ] + + @pytest.mark.parametrize("method", ["update_item_payload", "update_result_payload"]) def test_empty_payload_update_is_sent(api, method): experiment, send, respond = api @@ -181,6 +217,24 @@ def test_mutations_surface_errors_without_replaying( send.assert_called_once() +@pytest.mark.parametrize("method", ["update_result", "delete_result"]) +def test_lost_mutation_response_is_unknown_and_never_retried(api, method): + experiment, send, _ = api + send.side_effect = ( + httpx.ReadTimeout("lost response") + if isinstance(experiment, AsyncExperiment) + else requests.ReadTimeout("lost response") + ) + fields = {"value": True} if method == "update_result" else {} + with pytest.raises(EvaluationAPIError) as caught: + invoke(getattr(experiment, method), RESOURCE_ID, revision=1, **fields) + assert (caught.value.code, caught.value.persistence) == ( + "transport_error", + "unknown", + ) + send.assert_called_once() + + @pytest.mark.parametrize( "method,fields,status,response", [ @@ -209,6 +263,12 @@ def test_malformed_acknowledgments_have_unknown_outcome( [ ("update_item", RESOURCE_ID, {"revision": 0}), ("update_result", RESOURCE_ID, {"revision": 1, "value": 2**53 + 1}), + ("update_result", RESOURCE_ID, {"revision": 0, "status": "ok", "value": None}), + ( + "update_result", + RESOURCE_ID, + {"revision": 0, "value": False, "error_code": "timeout"}, + ), ( "update_result", RESOURCE_ID,