From aaeac5399fdc6afa1bf5bd07ac660b698fcc2b39 Mon Sep 17 00:00:00 2001 From: Dave Horton Date: Sat, 24 Oct 2026 16:53:11 -0400 Subject: [PATCH] feat(agent): speak, speechCleanup, noResponseMaxPrompts; schema v0.4.26 - bundled schema synced to @jambonz/schema v0.4.26 - verb_builder.pyi regenerated - AgentVerb TypedDict: speak, speechCleanup, noResponseMaxPrompts Co-Authored-By: Claude Opus 5.5 --- scripts/sync_schema.py | 2 +- .../recognizer-mistralOptions.schema.json | 86 +++++++++ .../schema/components/recognizer.schema.json | 3 +- .../schema/components/synthesizer.schema.json | 2 +- .../schema/jambonz-app.schema.json | 1 + .../schema/verbs/agent.schema.json | 42 +++- .../schema/verbs/config.schema.json | 18 ++ .../schema/verbs/nova_s2s.schema.json | 179 ++++++++++++++++++ src/jambonz_sdk/types/verbs.py | 3 + src/jambonz_sdk/verb_builder.pyi | 6 + 10 files changed, 335 insertions(+), 7 deletions(-) create mode 100644 src/jambonz_sdk/schema/components/recognizer-mistralOptions.schema.json create mode 100644 src/jambonz_sdk/schema/verbs/nova_s2s.schema.json diff --git a/scripts/sync_schema.py b/scripts/sync_schema.py index a191a5f..db4b07a 100644 --- a/scripts/sync_schema.py +++ b/scripts/sync_schema.py @@ -22,7 +22,7 @@ from pathlib import Path # ── Pin the schema version here ────────────────────────────────────── -SCHEMA_VERSION = "v0.4.23" +SCHEMA_VERSION = "v0.4.26" # ──────────────────────────────────────────────────────────────────── DEST = Path(__file__).resolve().parent.parent / "src" / "jambonz_sdk" / "schema" diff --git a/src/jambonz_sdk/schema/components/recognizer-mistralOptions.schema.json b/src/jambonz_sdk/schema/components/recognizer-mistralOptions.schema.json new file mode 100644 index 0000000..a6e4e11 --- /dev/null +++ b/src/jambonz_sdk/schema/components/recognizer-mistralOptions.schema.json @@ -0,0 +1,86 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://jambonz.org/schema/components/recognizer-mistralOptions", + "title": "Mistral Recognizer Options", + "description": "Mistral Voxtral realtime STT options. Only applies when recognizer vendor is 'mistral'. Voxtral streams text as it is recognized (sent as interim results when interim is enabled); with turn detection on (the default) a final transcript is produced when Voxtral detects the end of the speaker's turn, and in the agent verb each final ends the user's turn. Transcripts are reported with confidence 1.0. The language is sent as a two-letter code (en-US becomes en); leave it unset or 'any' for automatic detection.", + "type": "object", + "properties": { + "apiKey": { + "type": "string", + "description": "Mistral API key. Overrides the key configured in jambonz." + }, + "model": { + "type": "string", + "description": "Realtime transcription model. Defaults to voxtral-mini-transcribe-realtime-3.", + "examples": ["voxtral-mini-transcribe-realtime-3"] + }, + "streamingDelayMs": { + "type": "integer", + "description": "Target streaming delay in ms: how far the model looks ahead before emitting text. Lower is faster, higher is more accurate. Any value from 80 to 1200, or 2400. Voxtral's default is 480.", + "anyOf": [ + { "minimum": 80, "maximum": 1200 }, + { "const": 2400 } + ] + }, + "verbatim": { + "type": "boolean", + "description": "Ask Voxtral for a verbatim transcript (sent as verbatim). Not in Mistral's docs yet; the realtime API accepts it." + }, + "contextBias": { + "type": "array", + "items": { "type": "string" }, + "maxItems": 100, + "description": "Words or phrases to favor (names, brands, jargon). When omitted, recognizer.hints are used." + }, + "diarize": { + "type": "boolean", + "description": "Label each transcript with a speaker. Voxtral cannot diarize and detect turns at once, so this turns off turn detection: a final transcript closes at each sentence end or change of speaker. recognizer.diarization has the same effect. Ignored in the agent verb, which needs turn detection." + }, + "speakerReferences": { + "type": "array", + "maxItems": 8, + "description": "Reference clips (3 to 15 seconds each) that let diarization name known speakers. Passed to Mistral as speaker_references. Requires diarize.", + "items": { "type": "object" } + }, + "turnDetection": { + "type": "object", + "description": "Tunes when Voxtral ends a turn. Ignored when diarize is on. Thresholds must satisfy startThreshold >= eagerEndThreshold > endThreshold.", + "properties": { + "startThreshold": { + "type": "number", + "minimum": 0, + "maximum": 1, + "description": "Speech probability that starts a turn. Default 0.5." + }, + "eagerEndThreshold": { + "type": "number", + "minimum": 0, + "maximum": 1, + "description": "Probability at which Voxtral signals a likely end of turn early; the agent verb uses it for early generation. Default 0.5." + }, + "endThreshold": { + "type": "number", + "minimum": 0, + "maximum": 1, + "description": "Probability below which silence counts toward ending the turn. Default 0.33." + }, + "minSilenceMs": { + "type": "integer", + "minimum": 0, + "description": "Silence in ms before a turn can end. Default 500; raised to streamingDelayMs when that is longer, since Voxtral's text lags the audio by about that much." + }, + "endTimeoutMs": { + "type": "integer", + "minimum": 0, + "description": "Silence in ms after which the turn ends regardless of the model. Default 3000; must be at least minSilenceMs." + } + }, + "additionalProperties": false + }, + "mistralSttUri": { + "type": "string", + "description": "Custom host to connect to instead of api.mistral.ai, e.g. a proxy. A ws:// or http:// prefix disables TLS." + } + }, + "additionalProperties": false +} diff --git a/src/jambonz_sdk/schema/components/recognizer.schema.json b/src/jambonz_sdk/schema/components/recognizer.schema.json index 8c7e1b3..85b10fe 100644 --- a/src/jambonz_sdk/schema/components/recognizer.schema.json +++ b/src/jambonz_sdk/schema/components/recognizer.schema.json @@ -8,7 +8,7 @@ "vendor": { "type": "string", "description": "The STT vendor to use. Must match a vendor configured in the jambonz platform.", - "examples": ["google", "aws", "microsoft", "deepgram", "ibm", "nvidia", "soniox", "xai", "modulate", "zoom", "gradium", "assemblyai", "speechmatics", "openai", "houndify", "gladia", "elevenlabs", "custom"] + "examples": ["google", "aws", "microsoft", "deepgram", "ibm", "nvidia", "soniox", "xai", "modulate", "zoom", "gradium", "mistral", "assemblyai", "speechmatics", "openai", "houndify", "gladia", "elevenlabs", "custom"] }, "label": { "type": "string", @@ -202,6 +202,7 @@ "modulateOptions": { "$ref": "recognizer-modulateOptions" }, "zoomOptions": { "$ref": "recognizer-zoomOptions" }, "gradiumOptions": { "$ref": "recognizer-gradiumOptions" }, + "mistralOptions": { "$ref": "recognizer-mistralOptions" }, "assemblyAiOptions": { "$ref": "recognizer-assemblyAiOptions" }, "speechmaticsOptions": { "$ref": "recognizer-speechmaticsOptions" }, "openaiOptions": { "$ref": "recognizer-openaiOptions" }, diff --git a/src/jambonz_sdk/schema/components/synthesizer.schema.json b/src/jambonz_sdk/schema/components/synthesizer.schema.json index c88ac54..f01982d 100644 --- a/src/jambonz_sdk/schema/components/synthesizer.schema.json +++ b/src/jambonz_sdk/schema/components/synthesizer.schema.json @@ -8,7 +8,7 @@ "vendor": { "type": "string", "description": "The TTS vendor to use. Must match a vendor configured in the jambonz platform.", - "examples": ["google", "aws", "microsoft", "elevenlabs", "cartesia", "fishaudio", "gradium", "kugelaudio", "nineninesix", "speechify", "deepgram", "deepgramflux", "ibm", "nvidia", "wellsaid", "whisper", "custom"] + "examples": ["google", "aws", "microsoft", "elevenlabs", "cartesia", "fishaudio", "gradium", "kugelaudio", "mistral", "nineninesix", "speechify", "deepgram", "deepgramflux", "ibm", "nvidia", "wellsaid", "whisper", "custom"] }, "label": { "type": "string", diff --git a/src/jambonz_sdk/schema/jambonz-app.schema.json b/src/jambonz_sdk/schema/jambonz-app.schema.json index a8af4bf..0763b25 100644 --- a/src/jambonz_sdk/schema/jambonz-app.schema.json +++ b/src/jambonz_sdk/schema/jambonz-app.schema.json @@ -30,6 +30,7 @@ { "$ref": "verbs/qwen_s2s" }, { "$ref": "verbs/gptlive_s2s" }, { "$ref": "verbs/voicelive_s2s" }, + { "$ref": "verbs/nova_s2s" }, { "$ref": "verbs/dialogflow" }, { "$ref": "verbs/agent" }, { "$ref": "verbs/transfer" }, diff --git a/src/jambonz_sdk/schema/verbs/agent.schema.json b/src/jambonz_sdk/schema/verbs/agent.schema.json index e38dcf4..c6767f1 100644 --- a/src/jambonz_sdk/schema/verbs/agent.schema.json +++ b/src/jambonz_sdk/schema/verbs/agent.schema.json @@ -26,12 +26,12 @@ { "type": "boolean" }, { "type": "string", "enum": ["always"] } ], - "description": "When using Deepgram Flux Multilingual, automatically adjust STT language hints and switch TTS voice based on detected language. Values: false (disabled), true (lock on first utterance), 'always' (continuously adapt on every turn). Default: false.", + "description": "Switch the TTS to the language the caller speaks, using the matching languageConfig entry. Works with any STT that reports the detected language: Deepgram (language 'multi', or Flux Multilingual), AssemblyAI (languageDetection), Google and Azure (altLanguages), Soniox (enableLanguageIdentification), Gladia and Modulate. Values: false (disabled), true (lock on the first utterance whose language is detected), 'always' (re-evaluate every turn). With Deepgram Flux, true also locks the STT language hints to the detected language. Default: false.", "default": false }, "languageConfig": { "type": "object", - "description": "Per-language overrides for TTS. Keys are BCP-47 language codes. When autoLockLanguage detects a language switch, the agent uses the corresponding config.", + "description": "Per-language TTS overrides, used when autoLockLanguage detects a language. Keys are language codes such as 'es' or 'es-MX'. A detected language matches an exact key first (case-insensitive), then a key with the same primary language, so 'es' matches a detected 'es-US'. A detected language with no entry keeps the current TTS. The entry's tts is merged over the agent's tts; set its language to the code the TTS vendor expects.", "additionalProperties": { "type": "object", "properties": { @@ -119,7 +119,13 @@ "noResponseTimeout": { "type": "number", "minimum": 0, - "description": "Seconds to wait after the assistant finishes speaking before prompting the user to respond. 0 disables. Default: 0.", + "description": "Seconds to wait after the assistant finishes speaking before prompting the user to respond. 0 disables. Default: 12.", + "default": 12 + }, + "noResponseMaxPrompts": { + "type": "integer", + "minimum": 0, + "description": "End the agent after this many no-response prompts in a row get no reply. The actionHook then fires with completionReason 'no_response', so the application decides what happens next. The count resets whenever the user speaks. 0 or unset: no limit (the agent keeps prompting). Requires feature-server with jambonz/feature-server#259.", "default": 0 }, "llm": { @@ -225,11 +231,21 @@ "description": "Enable speculative LLM prompting before end-of-turn is confirmed. When using Krisp turn detection, set this to true to speculatively prompt the LLM before Krisp confirms the turn has ended. If the transcript matches when turn ends, buffered tokens are released immediately — reducing response latency. Note: Deepgram Flux performs early generation automatically via its native EagerEndOfTurn signal regardless of this setting. Default: false.", "default": false }, + "speak": { + "type": "boolean", + "description": "Whether the agent's text is spoken. With false the agent never talks (for example while it drives a DTMF-only phone menu with its tools): the LLM's text is still kept in conversation history, llm_response events and the transcript, but is never sent to TTS, and toolFiller and earlyGeneration are turned off. Each reply ends when the LLM finishes, as for a reply that only called tools. Default: true.", + "default": true + }, + "speechCleanup": { + "type": "boolean", + "description": "Keep text the model was not meant to say out of TTS: a paragraph that is only a stage direction in (), [] or ** (such as '(silence)' or '[No response needed]'), and , or blocks. Asides inside a sentence are still spoken. Applied as the reply streams; ordinary speech is not delayed. A reply with nothing left to say ends without speaking. History, llm_response events and the transcript keep the model's raw text. Default: true.", + "default": true + }, "noiseIsolation": { "oneOf": [ { "type": "string", - "enum": ["krisp", "rnnoise"], + "enum": ["krisp", "rnnoise", "kugelaudio"], "description": "Shorthand — enable noise isolation with the specified vendor using default settings." }, { @@ -240,6 +256,24 @@ "type": "string", "description": "Noise isolation vendor/mode (e.g. 'krisp')." }, + "label": { + "type": "string", + "description": "Label of the speech credential to use, for vendors that need one (kugelaudio)." + }, + "auth": { + "type": "object", + "description": "Optional inline credentials for vendors that need them (kugelaudio). When omitted, feature-server looks up the speech credential by (vendor, label).", + "properties": { + "apiKey": { + "type": "string" + }, + "apiUri": { + "type": "string", + "description": "Vendor API host, e.g. api.eu.kugelaudio.com." + } + }, + "additionalProperties": true + }, "level": { "type": "number", "minimum": 0, diff --git a/src/jambonz_sdk/schema/verbs/config.schema.json b/src/jambonz_sdk/schema/verbs/config.schema.json index 4b09ace..546b799 100644 --- a/src/jambonz_sdk/schema/verbs/config.schema.json +++ b/src/jambonz_sdk/schema/verbs/config.schema.json @@ -176,6 +176,24 @@ }, "model": { "type": "string" + }, + "label": { + "type": "string", + "description": "Label of the speech credential to use, for vendors that need one (kugelaudio)." + }, + "auth": { + "type": "object", + "description": "Optional inline credentials for vendors that need them (kugelaudio). When omitted, feature-server looks up the speech credential by (vendor, label).", + "properties": { + "apiKey": { + "type": "string" + }, + "apiUri": { + "type": "string", + "description": "Vendor API host, e.g. api.eu.kugelaudio.com." + } + }, + "additionalProperties": true } } }, diff --git a/src/jambonz_sdk/schema/verbs/nova_s2s.schema.json b/src/jambonz_sdk/schema/verbs/nova_s2s.schema.json new file mode 100644 index 0000000..d79d77f --- /dev/null +++ b/src/jambonz_sdk/schema/verbs/nova_s2s.schema.json @@ -0,0 +1,179 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://jambonz.org/schema/verbs/nova_s2s", + "minVersion": "11.1.7", + "title": "Amazon Nova Sonic S2S", + "description": "Shortcut for 'llm' with vendor automatically set to 'nova'. Connects the caller to Amazon Nova Sonic (Nova 2 / Nova 2.5 Sonic) on Amazon Bedrock for real-time speech-to-speech conversation. auth takes AWS IAM credentials (accessKeyId, secretAccessKey, optional sessionToken) and the Bedrock region; the credentials need bedrock:InvokeModelWithBidirectionalStream on the model. llmOptions is optional: systemPrompt, history, greeting, and sessionStart / promptStart bodies merged into Nova's own events (the audio formats are fixed by jambonz). Requires the mediajam media server.", + "type": "object", + "allOf": [ + { + "$ref": "../components/llm-base" + } + ], + "properties": { + "verb": { + "const": "nova_s2s", + "description": "The verb name." + }, + "vendor": { + "type": "string", + "const": "nova", + "description": "The LLM vendor (always 'nova' for this shortcut)." + }, + "model": { + "type": "string", + "description": "The Bedrock model id. Defaults to amazon.nova-2-5-sonic.", + "examples": [ + "amazon.nova-2-5-sonic", + "amazon.nova-2-sonic-v1:0" + ] + }, + "auth": { + "type": "object", + "description": "AWS IAM credentials and the Bedrock region.", + "properties": { + "accessKeyId": { + "type": "string", + "minLength": 1, + "description": "AWS access key id." + }, + "secretAccessKey": { + "type": "string", + "minLength": 1, + "description": "AWS secret access key." + }, + "sessionToken": { + "type": "string", + "description": "Session token, for temporary (STS) credentials." + }, + "region": { + "type": "string", + "description": "Bedrock region serving Nova Sonic. Defaults to us-east-1.", + "examples": [ + "us-east-1", + "us-west-2", + "eu-north-1", + "ap-northeast-1" + ] + } + }, + "required": [ + "accessKeyId", + "secretAccessKey" + ], + "additionalProperties": true + }, + "llmOptions": { + "type": "object", + "description": "Nova session options. A systemPrompt or other text above 50KB is split across several textInput events.", + "properties": { + "systemPrompt": { + "type": "string", + "description": "System instructions, sent as the SYSTEM text content." + }, + "speechPrompt": { + "type": "string", + "description": "Sent after the system prompt with role SYSTEM_SPEECH. Nova accepts one of three fixed sentences that choose Latin, Devanagari or mixed script for Hindi transcription and responses." + }, + "history": { + "type": "array", + "description": "Prior turns sent before the audio starts. Following Nova's rules, consecutive turns with the same role are merged, leading assistant turns are dropped, and the total is capped at 200KB.", + "items": { + "type": "object", + "properties": { + "role": { + "type": "string", + "enum": [ + "user", + "assistant", + "USER", + "ASSISTANT" + ] + }, + "content": { + "type": "string" + } + }, + "required": [ + "role", + "content" + ] + } + }, + "greeting": { + "type": "string", + "description": "Sent as a user text turn once the audio starts, so Nova speaks first (Nova otherwise waits for the caller)." + }, + "sessionStart": { + "type": "object", + "description": "Merged into Nova's sessionStart event: inferenceConfiguration, turnDetectionConfiguration.endpointingSensitivity (HIGH, MEDIUM, LOW)." + }, + "promptStart": { + "type": "object", + "description": "Merged into Nova's promptStart event: audioOutputConfiguration.voiceId (default matthew), toolConfiguration.tools in Nova's toolSpec shape and toolConfiguration.toolChoice ({auto:{}}, {any:{}} or {tool:{name}}). The audio format fields are set by jambonz." + }, + "dtmf": { + "type": "object", + "description": "Caller keypad input. Nova has no DTMF support, so each completed entry is sent as a user text turn. Takes the gather verb's digit options.", + "properties": { + "enable": { + "type": "boolean" + }, + "finishOnKey": { + "type": "string" + }, + "numDigits": { + "type": "number" + }, + "minDigits": { + "type": "number" + }, + "maxDigits": { + "type": "number" + }, + "interDigitTimeout": { + "type": "number" + }, + "timeout": { + "type": "number" + } + }, + "required": [ + "enable" + ] + } + }, + "additionalProperties": true + } + }, + "required": [ + "auth" + ], + "examples": [ + { + "verb": "nova_s2s", + "model": "amazon.nova-2-5-sonic", + "auth": { + "accessKeyId": "AKIA...", + "secretAccessKey": "...", + "region": "us-east-1" + }, + "llmOptions": { + "systemPrompt": "You are a helpful voice assistant. Keep answers short.", + "greeting": "Greet the caller and ask how you can help.", + "sessionStart": { + "turnDetectionConfiguration": { + "endpointingSensitivity": "MEDIUM" + } + }, + "promptStart": { + "audioOutputConfiguration": { + "voiceId": "tiffany" + } + } + }, + "toolHook": "/tool-call", + "actionHook": "/s2s-complete" + } + ] +} diff --git a/src/jambonz_sdk/types/verbs.py b/src/jambonz_sdk/types/verbs.py index dcbcc38..efbdacc 100644 --- a/src/jambonz_sdk/types/verbs.py +++ b/src/jambonz_sdk/types/verbs.py @@ -524,12 +524,15 @@ class AgentVerb(TypedDict, total=False): turnDetection: str | dict[str, Any] bargeIn: dict[str, Any] noResponseTimeout: int + noResponseMaxPrompts: int llm: dict[str, Any] actionHook: ActionHook eventHook: ActionHook toolHook: ActionHook greeting: bool earlyGeneration: bool + speak: bool + speechCleanup: bool noiseIsolation: str | dict[str, Any] mcpServers: list[McpServer] diff --git a/src/jambonz_sdk/verb_builder.pyi b/src/jambonz_sdk/verb_builder.pyi index d2cbd2e..7f96263 100644 --- a/src/jambonz_sdk/verb_builder.pyi +++ b/src/jambonz_sdk/verb_builder.pyi @@ -366,12 +366,15 @@ class VerbBuilder: turnDetection: Any = ..., bargeIn: dict[str, Any] = ..., noResponseTimeout: int | float = ..., + noResponseMaxPrompts: Any = ..., llm: dict[str, Any] = ..., actionHook: Any = ..., eventHook: Any = ..., toolHook: Any = ..., greeting: bool = ..., earlyGeneration: bool = ..., + speak: bool = ..., + speechCleanup: bool = ..., noiseIsolation: Any = ..., mcpServers: list[Any] = ..., handoff: Any = ..., @@ -391,12 +394,15 @@ class VerbBuilder: turnDetection: Any bargeIn: dict[str, Any] noResponseTimeout: int | float + noResponseMaxPrompts: Any llm: dict[str, Any] actionHook: Any eventHook: Any toolHook: Any greeting: bool earlyGeneration: bool + speak: bool + speechCleanup: bool noiseIsolation: Any mcpServers: list[Any] handoff: Any