[Brook G2] Converted transfer types nested in JSON values.

changelog/changed/nesting-matryoshka.md

@@ -0,0 +1 @@
+Convert values of a `transfer_type_convertible` class inside JSON values too, such as list items and dataclass fields, not only at the top level of a payload. A dataclass with the decorator is still written as its fields inside a JSON value.

temporalio/converter/_payload_converter.py

@@ -110,6 +110,11 @@ def transfer_type_convertible(
 ) -> _TransferTypeConvertibleDecorator[ValueT, TransferTypeT]:
     """Decorate a class with a transfer type converter class.
 
+    The converter converts a value of the class to its transfer type at the top level of a
+    payload, and inside a JSON value such as a list item or a dataclass field. Inside a JSON
+    value, a dataclass is the exception. ``dataclasses.asdict`` writes it as its fields, and
+    the converter decodes it from them.
+
     .. warning::
         This API is experimental and subject to change.
     """
@@ -125,6 +130,21 @@ def _get_transfer_type_converter(
     return None
 
 
+def _nested_transfer_type_converter(
+    hint: object,
+) -> TransferTypeConverter[Any, Any] | None:
+    # The JSON converter calls this for each value that isn't a primitive, so the fastest check
+    # is first. A generic alias gives the attribute of its origin class.
+    if getattr(hint, _TRANSFER_TYPE_CONVERTER_ATTR, None) is None:
+        return None
+    origin = typing.get_origin(hint) or hint
+    # dataclasses.asdict writes a nested dataclass as its fields, so only a class that isn't a
+    # dataclass is converted there.
+    if not inspect.isclass(origin) or dataclasses.is_dataclass(origin):
+        return None
+    return _get_transfer_type_converter(origin)
+
+
 class PayloadConverter(ABC):
     """Base payload converter to/from multiple payloads/values."""
 
@@ -677,6 +697,13 @@ def default(self, o: Any) -> Any:
         # Dataclass support
         if dataclasses.is_dataclass(o) and not isinstance(o, type):
             return dataclasses.asdict(o)
+        # Nested transfer type support
+        converter = _nested_transfer_type_converter(type(o))
+        if converter is not None:
+            value = converter.to_transfer_type(o)
+            if isinstance(value, google.protobuf.message.Message):
+                return google.protobuf.json_format.MessageToDict(value)
+            return value
         # Support for Pydantic v1's dict method
         dict_fn = getattr(o, "dict", None)
         if callable(dict_fn):
@@ -885,6 +912,20 @@ def value_to_type(
     if supertype:
         return value_to_type(supertype, value, custom_converters)
 
+    # Nested transfer type values, as AdvancedJSONEncoder writes them
+    converter = _nested_transfer_type_converter(hint)
+    if converter is not None:
+        transfer_type = converter.transfer_type
+        if inspect.isclass(transfer_type) and issubclass(
+            transfer_type, google.protobuf.message.Message
+        ):
+            value = google.protobuf.json_format.ParseDict(
+                value, transfer_type(), ignore_unknown_fields=True
+            )
+        elif transfer_type is not None:
+            value = value_to_type(transfer_type, value, custom_converters)
+        return converter.from_transfer_type(value, hint)
+
     # Load origin for other checks
     origin = getattr(hint, "__origin__", hint)
     type_args: tuple = getattr(hint, "__args__", ())

tests/test_converter.py

@@ -3,6 +3,7 @@
 import dataclasses
 import inspect
 import ipaddress
+import json
 import logging
 import sys
 import traceback
@@ -461,6 +462,112 @@ def test_temporal_transfer_type_payload_converter_with_generic_value(
     assert converter.from_payload(payload, type_hint) == value
 
 
+class NestedTransferTypeValueConverter(
+    TransferTypeConverter["NestedTransferTypeValue[Any]", dict[str, str]]
+):
+    transfer_type: type[dict[str, str]] | None = dict
+
+    def to_transfer_type(self, value: NestedTransferTypeValue[Any]) -> dict[str, str]:
+        return {"wireName": value.name}
+
+    def from_transfer_type(
+        self, value: dict[str, str], type_hint: type[NestedTransferTypeValue[Any]]
+    ) -> NestedTransferTypeValue[Any]:
+        args = typing.get_args(type_hint)
+        return NestedTransferTypeValue(value["wireName"], args[0] if args else None)
+
+
+class NestedTransferTypeValue(Generic[T]):
+    def __init__(self, name: str, kind: Any = None) -> None:
+        self.name = name
+        self.kind = kind
+
+
+transfer_type_convertible(NestedTransferTypeValueConverter)(NestedTransferTypeValue)
+
+
+@dataclass
+class HasNestedTransferTypeValues:
+    one: NestedTransferTypeValue[int]
+    many: list[NestedTransferTypeValue[str]]
+    proto: TemporalTransferTypeValueWithoutHint
+
+
+def test_transfer_type_values_nested_in_json():
+    converter = DataConverter.default.payload_converter
+    value = HasNestedTransferTypeValues(
+        one=NestedTransferTypeValue("a"),
+        many=[NestedTransferTypeValue("b")],
+        proto=TemporalTransferTypeValueWithoutHint("c"),
+    )
+    payload = converter.to_payload(value)
+    assert json.loads(payload.data) == {
+        "one": {"wireName": "a"},
+        "many": [{"wireName": "b"}],
+        "proto": {"value": "c"},
+    }
+    decoded = converter.from_payload(payload, HasNestedTransferTypeValues)
+    assert decoded.one.name == "a" and decoded.one.kind is int
+    assert decoded.many[0].name == "b" and decoded.many[0].kind is str
+    assert decoded.proto == TemporalTransferTypeValueWithoutHint("c")
+
+
+def test_transfer_type_dataclass_in_a_top_level_list_keeps_its_fields():
+    # The decode of a dataclass inside a JSON value reads its fields, so the encode writes them.
+    converter = DataConverter.default.payload_converter
+    value = [TemporalTransferTypeValueWithoutHint("c")]
+    payload = converter.to_payload(value)
+    assert json.loads(payload.data) == [{"value": "c"}]
+    decoded = converter.from_payload(
+        payload, list[TemporalTransferTypeValueWithoutHint]
+    )
+    assert decoded == value
+
+
+class NestedProtoValueConverter(
+    TransferTypeConverter[
+        "NestedProtoValue", temporalio.api.common.v1.WorkflowExecution
+    ]
+):
+    transfer_type: type[temporalio.api.common.v1.WorkflowExecution] | None = (
+        temporalio.api.common.v1.WorkflowExecution
+    )
+
+    def to_transfer_type(
+        self, value: NestedProtoValue
+    ) -> temporalio.api.common.v1.WorkflowExecution:
+        return temporalio.api.common.v1.WorkflowExecution(workflow_id=value.workflow_id)
+
+    def from_transfer_type(
+        self,
+        value: temporalio.api.common.v1.WorkflowExecution,
+        type_hint: type[NestedProtoValue],
+    ) -> NestedProtoValue:
+        return NestedProtoValue(value.workflow_id)
+
+
+class NestedProtoValue:
+    def __init__(self, workflow_id: str) -> None:
+        self.workflow_id = workflow_id
+
+    def __eq__(self, other: object) -> bool:
+        return (
+            isinstance(other, NestedProtoValue)
+            and other.workflow_id == self.workflow_id
+        )
+
+
+transfer_type_convertible(NestedProtoValueConverter)(NestedProtoValue)
+
+
+def test_proto_transfer_type_nested_in_json_is_its_proto_json():
+    converter = DataConverter.default.payload_converter
+    payload = converter.to_payload([NestedProtoValue("wf-1")])
+    assert json.loads(payload.data) == [{"workflowId": "wf-1"}]
+    decoded = converter.from_payload(payload, list[NestedProtoValue])
+    assert decoded == [NestedProtoValue("wf-1")]
+
+
 def test_transfer_type_convertible_rejects_existing_converter():
     with pytest.raises(TypeError, match="already has a transfer type converter"):
         transfer_type_convertible(TemporalTransferTypeValueConverter)(