feat(narratives): replace the parallel chart config with unified synthesis and block streaming

narratives/AGENTS.md

@@ -74,8 +74,7 @@ process and PR expectations: [`CONTRIBUTING.md`](../CONTRIBUTING.md).
       the default chart floor (`chart_defaults.py`),
       the synthesis prompt builder (`synthesis.py`),
       the streaming block splitter (`blocks.py`),
-      chart configuration (`chart_config.py`), follow-up generation
-      (`follow_up.py`),
+      follow-up generation (`follow_up.py`),
       user-visible messages (`messages.py`), the typed SSE frames the agent
       loop sends (`sse_frames.py`), the turn's state slots
       (`state_slots.py`), and signed transcripts (`transcript.py`). Of the

narratives/README.md

@@ -10,7 +10,7 @@ Context Protocol (MCP).
 
 - **Grounded statistical Q&A:** Answers natural-language questions using Data
   Commons MCP tools.
-- **Interactive visualizations:** Renders Data Commons charts alongside
+- **Interactive visualizations:** Renders Data Commons charts inline within
   generated responses.
 - **Source attribution:** Attaches inline citations and dataset provenance to
   statistical claims.

narratives/agent/src/narratives_agent/config_model.py

@@ -60,8 +60,6 @@ class GeminiRole(Enum):
     CLASSIFY = "classify"
     DISCOVERY = "discovery"
     SYNTHESIS = "synthesis"
-    CHART_CONFIG = "chart_config"
-    CHART_VALIDATION = "chart_validation"
     FOLLOW_UP = "follow_up"
     COMPACTION = "compaction"
 
@@ -85,8 +83,6 @@ class RoleConfig:
     GeminiRole.CLASSIFY: 0.0,
     GeminiRole.DISCOVERY: 0.2,
     GeminiRole.SYNTHESIS: 0.3,
-    GeminiRole.CHART_CONFIG: 0.2,
-    GeminiRole.CHART_VALIDATION: 0.0,
     # A higher temperature varies the phrasing of the follow-up questions.
     GeminiRole.FOLLOW_UP: 0.8,
     GeminiRole.COMPACTION: 0.1,
@@ -100,8 +96,6 @@ class RoleConfig:
 # accept `MINIMAL`.
 _FIXED_THINKING_LEVELS: Mapping[GeminiRole, types.ThinkingLevel] = {
     GeminiRole.CLASSIFY: types.ThinkingLevel.MINIMAL,
-    GeminiRole.CHART_CONFIG: types.ThinkingLevel.LOW,
-    GeminiRole.CHART_VALIDATION: types.ThinkingLevel.LOW,
     GeminiRole.FOLLOW_UP: types.ThinkingLevel.LOW,
     GeminiRole.COMPACTION: types.ThinkingLevel.LOW,
 }
@@ -179,8 +173,6 @@ class GeminiConfig:
     classify_model: str = DEFAULT_GEMINI_CLASSIFY_MODEL
     discovery_model: str = DEFAULT_GEMINI_MODEL
     synthesis_model: str = DEFAULT_GEMINI_MODEL
-    chart_config_model: str = DEFAULT_GEMINI_MODEL
-    chart_validation_model: str = DEFAULT_GEMINI_MODEL
     follow_up_model: str = DEFAULT_GEMINI_MODEL
     compaction_model: str = DEFAULT_GEMINI_MODEL
     api_key: str = field(default="", repr=False)
@@ -445,14 +437,6 @@ def parse_agent_config(document: Mapping[str, Any]) -> AgentConfig:
             synthesis_model=_read_model_name(
                 gemini, "synthesis_model", GeminiConfig.synthesis_model
             ),
-            chart_config_model=_read_model_name(
-                gemini, "chart_config_model", GeminiConfig.chart_config_model
-            ),
-            chart_validation_model=_read_model_name(
-                gemini,
-                "chart_validation_model",
-                GeminiConfig.chart_validation_model,
-            ),
             follow_up_model=_read_model_name(
                 gemini, "follow_up_model", GeminiConfig.follow_up_model
             ),

narratives/agent/src/narratives_agent/config_model_test.py

@@ -77,8 +77,6 @@ def test_an_empty_document_gives_every_default() -> None:
         classify_model=DEFAULT_GEMINI_CLASSIFY_MODEL,
         discovery_model=DEFAULT_GEMINI_MODEL,
         synthesis_model=DEFAULT_GEMINI_MODEL,
-        chart_config_model=DEFAULT_GEMINI_MODEL,
-        chart_validation_model=DEFAULT_GEMINI_MODEL,
         follow_up_model=DEFAULT_GEMINI_MODEL,
         compaction_model=DEFAULT_GEMINI_MODEL,
     )
@@ -104,8 +102,6 @@ def test_configured_values_are_carried_through() -> None:
                 "classify_model": " gemini-3.8-flash ",
                 "discovery_model": " gemini-3-pro ",
                 "synthesis_model": " gemini-3-pro-synthesis ",
-                "chart_config_model": " gemini-3-flash-charts ",
-                "chart_validation_model": " gemini-3-flash-val ",
                 "follow_up_model": " gemini-3-flash-followup ",
                 "compaction_model": " gemini-3-flash-compact ",
                 "api_key": " test-key ",
@@ -130,8 +126,6 @@ def test_configured_values_are_carried_through() -> None:
             classify_model="gemini-3.8-flash",
             discovery_model="gemini-3-pro",
             synthesis_model="gemini-3-pro-synthesis",
-            chart_config_model="gemini-3-flash-charts",
-            chart_validation_model="gemini-3-flash-val",
             follow_up_model="gemini-3-flash-followup",
             compaction_model="gemini-3-flash-compact",
             api_key="test-key",
@@ -361,16 +355,6 @@ def test_an_unrecognized_thinking_level_gives_the_default(
             RoleConfig(DEFAULT_GEMINI_MODEL, 0.3, types.ThinkingLevel.MEDIUM),
             id="synthesis",
         ),
-        pytest.param(
-            GeminiRole.CHART_CONFIG,
-            RoleConfig(DEFAULT_GEMINI_MODEL, 0.2, types.ThinkingLevel.LOW),
-            id="chart_config",
-        ),
-        pytest.param(
-            GeminiRole.CHART_VALIDATION,
-            RoleConfig(DEFAULT_GEMINI_MODEL, 0.0, types.ThinkingLevel.LOW),
-            id="chart_validation",
-        ),
         pytest.param(
             GeminiRole.FOLLOW_UP,
             RoleConfig(DEFAULT_GEMINI_MODEL, 0.8, types.ThinkingLevel.LOW),
@@ -446,8 +430,6 @@ def test_configured_values_apply_to_the_roles_they_govern() -> None:
             classify_model="gemini-3.6-flash",
             discovery_model="gemini-discovery-custom",
             synthesis_model="gemini-synthesis-custom",
-            chart_config_model="gemini-chart-config-custom",
-            chart_validation_model="gemini-chart-validation-custom",
             follow_up_model="gemini-follow-up-custom",
             compaction_model="gemini-compaction-custom",
         ),
@@ -466,14 +448,6 @@ def test_configured_values_apply_to_the_roles_they_govern() -> None:
     assert config.resolve_role_config(GeminiRole.SYNTHESIS) == RoleConfig(
         "gemini-synthesis-custom", 0.3, types.ThinkingLevel.LOW
     )
-    assert config.resolve_role_config(GeminiRole.CHART_CONFIG) == RoleConfig(
-        "gemini-chart-config-custom", 0.2, types.ThinkingLevel.LOW
-    )
-    assert config.resolve_role_config(
-        GeminiRole.CHART_VALIDATION
-    ) == RoleConfig(
-        "gemini-chart-validation-custom", 0.0, types.ThinkingLevel.LOW
-    )
     assert config.resolve_role_config(GeminiRole.FOLLOW_UP) == RoleConfig(
         "gemini-follow-up-custom", 0.8, types.ThinkingLevel.LOW
     )

narratives/agent/src/narratives_agent/gemini/client_test.py

@@ -41,8 +41,6 @@
     StreamChunk,
 )
 from narratives_agent.gemini.schemas import (
-    ChartConfigResponse,
-    DataValidationResponse,
     FollowUpResponse,
     QueryClassification,
 )
@@ -440,37 +438,37 @@ async def test_a_structured_output_request_sends_the_schema_and_config(
     gemini: _FakeGemini,
 ) -> None:
     # Test: Request configuration for a structured-output call.
-    # Situation: A caller passes ChartConfigResponse as response_schema with a
+    # Situation: A caller passes FollowUpResponse as response_schema with a
     #   system instruction, temperature 0.2, and `ThinkingLevel.LOW`.
     # Expectation: The request sets responseMimeType to "application/json"
     #   with the converted responseSchema, includes the rendered system
     #   instruction and generation settings, and returns a `GeminiResponse`
     #   whose text is the structured reply.
-    answer = json.dumps({"should_render": True, "charts": []})
+    answer = json.dumps({"questions": ["And Spain?"]})
     gemini.respond(
         _answer(200, json=_response_json([{"text": answer}], _USAGE))
     )
 
     result = await _call(
         "async_request",
-        system_instruction="Extract charts.",
+        system_instruction="Suggest questions.",
         temperature=0.2,
         thinking_level=types.ThinkingLevel.LOW,
-        response_schema=ChartConfigResponse,
+        response_schema=FollowUpResponse,
     )
 
     config = gemini.body()["generationConfig"]
     assert config["responseMimeType"] == "application/json"
-    assert config["responseSchema"]["required"] == ["should_render"]
+    assert config["responseSchema"]["required"] == ["questions"]
     assert config["temperature"] == 0.2
     # The SDK serializes ThinkingConfig fields in snake_case, which the
     # Gemini API's protobuf JSON parser accepts alongside camelCase.
     assert config["thinkingConfig"] == {"thinking_level": "LOW"}
     assert gemini.body()["systemInstruction"]["parts"] == [
-        {"text": "rendered:Extract charts."}
+        {"text": "rendered:Suggest questions."}
     ]
     assert isinstance(result, GeminiResponse)
-    assert json.loads(result.text) == {"should_render": True, "charts": []}
+    assert json.loads(result.text) == {"questions": ["And Spain?"]}
 
 
 @pytest.mark.asyncio
@@ -602,16 +600,14 @@ async def test_an_api_key_crossing_the_error_length_limit_is_redacted(
     ("schema", "required"),
     [
         (QueryClassification, ["serviceable"]),
-        (ChartConfigResponse, ["should_render"]),
-        (DataValidationResponse, ["data_found"]),
         (FollowUpResponse, ["questions"]),
     ],
 )
 async def test_each_schema_is_sent_to_gemini_as_expected(
     gemini: _FakeGemini, schema: type[BaseModel], required: list[str]
 ) -> None:
     # Test: The response schema each workflow sends on the wire.
-    # Situation: A structured-output request is made with each of the four
+    # Situation: A structured-output request is made with each of the two
     #   workflow schemas.
     # Expectation: The request carries the required fields and typed
     #   properties, and omits additional_properties (which the Gemini
@@ -629,42 +625,6 @@ async def test_each_schema_is_sent_to_gemini_as_expected(
     assert set(sent["properties"]) == set(schema.model_fields)
 
 
-@pytest.mark.asyncio
-async def test_the_chart_schema_constrains_each_chart_on_the_wire(
-    gemini: _FakeGemini,
-) -> None:
-    # Test: The nested chart item in the chart config schema.
-    # Situation: A request is made with the chart config schema.
-    # Expectation: Each chart item requires a title, enumerates only the
-    #   supported chart types, marks optional fields nullable, and omits
-    #   additional_properties.
-    gemini.respond(_answer(200, json=_response_json([{"text": "{}"}])))
-
-    await client.async_gemini_request(
-        _MESSAGES, "", _MODEL, response_schema=ChartConfigResponse
-    )
-
-    charts = gemini.body()["generationConfig"]["responseSchema"]["properties"][
-        "charts"
-    ]
-    item = charts["items"]
-    assert charts["nullable"] is True
-    assert item["required"] == ["title"]
-    assert "additional_properties" not in item
-    assert item["properties"]["viz_type"]["enum"] == [
-        "line",
-        "bar",
-        "ranking",
-        "pie",
-        "highlight",
-        "gauge",
-        "scatter",
-        "slider",
-    ]
-    assert item["properties"]["viz_type"]["nullable"] is True
-    assert item["properties"]["date"]["nullable"] is True
-
-
 @pytest.mark.asyncio
 @pytest.mark.parametrize("mode", ["async_request", "async_thought_streaming"])
 @pytest.mark.parametrize(

narratives/agent/src/narratives_agent/gemini/schemas.py

@@ -22,17 +22,6 @@
 
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
-ChartVizType = Literal[
-    "line",
-    "bar",
-    "ranking",
-    "pie",
-    "highlight",
-    "gauge",
-    "scatter",
-    "slider",
-]
-
 QueryPattern = Literal[
     "single_point",
     "top_n_ranking",
@@ -137,58 +126,6 @@ def _check_refusal_consistency(self) -> Self:
         return self
 
 
-class ChartItem(BaseModel):
-    """One chart configuration within a chart config response."""
-
-    model_config = _SCHEMA_CONFIG
-
-    viz_type: ChartVizType | None = None
-    # Required to prevent blank UI headers: forces the model to always
-    # generate a descriptive title rather than rendering an empty header strip.
-    # Asking for a title in the prompt alone was not enough; the model dropped
-    # it often enough to notice.
-    title: str = Field(description="Descriptive chart title")
-    variable_dcids: list[str] | None = None
-    place_dcids: list[str] | None = None
-    parent_place: str | None = None
-    child_place_type: str | None = None
-    date: str | None = Field(
-        default=None,
-        description="Single comparison date in YYYY, YYYY-MM, or YYYY-MM-DD",
-    )
-
-
-class ChartConfigResponse(BaseModel):
-    """Chart configurations, one per group of compatible variables."""
-
-    model_config = _SCHEMA_CONFIG
-
-    should_render: bool = Field(
-        description="True if at least one chart should be rendered"
-    )
-    charts: list[ChartItem] | None = Field(
-        default=None,
-        description=(
-            "Array of chart configurations (max 3). Group compatible "
-            "variables together."
-        ),
-    )
-
-
-class DataValidationResponse(BaseModel):
-    """Verdict on whether a synthesized answer contains actual data."""
-
-    model_config = _SCHEMA_CONFIG
-
-    data_found: bool = Field(
-        description=(
-            "True if the response contains actual data/statistics that "
-            "answer the query. False if data is unavailable, not found, "
-            "or the response says data doesn't exist."
-        )
-    )
-
-
 class FollowUpResponse(BaseModel):
     """Suggested follow-up questions for a completed answer."""
 
@@ -199,7 +136,5 @@ class FollowUpResponse(BaseModel):
     )
 
 
-# The workflows pass these names as `response_schema`.
-CHART_CONFIG_SCHEMA = ChartConfigResponse
-DATA_VALIDATION_SCHEMA = DataValidationResponse
+# Passed by the follow-up workflow as `response_schema`.
 FOLLOW_UP_SCHEMA = FollowUpResponse