diff --git a/src/Cli/Commands/Repl/ReplCommand.cs b/src/Cli/Commands/Repl/ReplCommand.cs index 8af7d4e8..c5b43db7 100644 --- a/src/Cli/Commands/Repl/ReplCommand.cs +++ b/src/Cli/Commands/Repl/ReplCommand.cs @@ -260,8 +260,7 @@ private async Task RunAsync( { // A host that names the model (fuseraft-web, per account) needs no config file: --model plus // FUSERAFT_API_KEY is a complete setup. Nothing here is saved unless --save asks. - if (userCfg is null && !string.IsNullOrWhiteSpace(settings.Model)) - userCfg = new UserConfig { ModelId = settings.Model.Trim(), Provider = settings.Provider?.Trim() ?? string.Empty }; + userCfg = WithHostModel(userCfg, settings.Model, settings.Provider); // Running from VS Code. Prefer an API key explicitly injected by the // extension (FUSERAFT_API_KEY), then fall back to any legacy plaintext @@ -1015,6 +1014,23 @@ private async Task RunAsync( return null; } + /// + /// The config a host's --model completes: a new one when there's no config file, or the file's when it has settings + /// but no model yet, as after fuseraft settings set or the VS Code settings panel on a fresh install. + /// + internal static UserConfig? WithHostModel(UserConfig? userCfg, string? model, string? provider) + { + if (string.IsNullOrWhiteSpace(model)) return userCfg; + if (userCfg is null) + return new UserConfig { ModelId = model.Trim(), Provider = provider?.Trim() ?? string.Empty }; + if (string.IsNullOrWhiteSpace(userCfg.ModelId)) + { + userCfg.ModelId = model.Trim(); + if (!string.IsNullOrWhiteSpace(provider)) userCfg.Provider = provider.Trim(); + } + return userCfg; + } + private static string? ResolveModelId(ReplSettings settings, UserConfig? userCfg) { var modelId = settings.Model?.Trim(); diff --git a/src/Cli/Commands/Repl/ReplTurn.cs b/src/Cli/Commands/Repl/ReplTurn.cs index 1b42e1e6..da07c981 100644 --- a/src/Cli/Commands/Repl/ReplTurn.cs +++ b/src/Cli/Commands/Repl/ReplTurn.cs @@ -187,6 +187,11 @@ internal static void AppendStoppedEarlyNote(List history, string re return forced; } + // A provider's refusal of the forced tool choice BuildRequestOptions asked for, rather than of the request as a whole. + internal static bool IsToolChoiceRejection(Exception ex) => + ex is not OperationCanceledException && + $"{ex.Message} {ex.InnerException?.Message}".Contains("tool_choice", StringComparison.OrdinalIgnoreCase); + // The options for a reply that mustn't call tools. The tools stay declared, since a provider may reject a // history with tool calls in it when none are, and the model is told not to use them. internal static ChatOptions? WithoutToolCalls(ChatOptions? baseOptions) @@ -898,6 +903,17 @@ await ctx.Emitter.EmitAsync(EventTypes.UserInput, turn: ctx.TurnIndex, responseText = SanitizeAssistantResponse(responseText, out var warningMessage); if (!capturePlan && responseText.Length == 0 && !stoppedEarly) { + // The reply is dropped, but the tools it called have run. Keep their calls and results, so the retry below + // and later turns know what was done: told nothing, the model ran a refused write again, and would rerun + // a commit or a deploy the same way. + var kept = CompletedRounds(rawUpdates); + if (kept.Count > 0) + { + ctx.History.AddRange(kept); + RepairDanglingToolCalls(ctx.History); + AppendStoppedEarlyNote(ctx.History, "the reply after these tool calls was empty or not a user-facing answer, so it was dropped"); + } + // Gated on its own emptyResponseRetried flag rather than the shared isCorrectionTurn // flag — an empty response can legitimately occur mid-way through an in-progress // todo-correction chain (a nudge message can provoke a bare tool call with no @@ -1350,12 +1366,14 @@ internal static string SanitizeAssistantResponse(string responseText, out string // response's non-blank content, by character count rather than raw line count — a leaked // single tool result is often accompanied by a short structural wrapper (e.g. a "Tool // Calls]" header or a bare tool name line) that would otherwise dilute a naive per-line - // majority check below the threshold that actually caught the real #119 incident. + // majority check below the threshold that actually caught the real #119 incident. Lines in a code + // fence don't count as leaked: a model quoting a tool's error to the user puts it in one. private static bool IsDominatedByToolResultLines(string trimmed) { var totalChars = 0; var matchedChars = 0; var hasMatch = false; + var inFence = false; foreach (var rawLine in trimmed.Split('\n')) { @@ -1363,7 +1381,12 @@ private static bool IsDominatedByToolResultLines(string trimmed) if (line.Length == 0) continue; totalChars += line.Length; - if (ToolResultLinePattern.IsMatch(line)) + if (line.StartsWith("```", StringComparison.Ordinal) || line.StartsWith("~~~", StringComparison.Ordinal)) + { + inFence = !inFence; + continue; + } + if (!inFence && ToolResultLinePattern.IsMatch(line)) { hasMatch = true; matchedChars += line.Length; @@ -2051,6 +2074,12 @@ await KeepGoingAfterFailuresAsync(ctx, consecutiveToolFailures, lastToolFailureD ctx.ActiveCts = null; return TurnStreamResult.MakeFailed(toolCallsThisTurn, limits.MaxIdenticalToolCalls); } + catch (Exception ex) when (requestOptions?.ToolMode is RequiredChatToolMode && rawUpdates.Count == 0 && IsToolChoiceRejection(ex)) + { + // Some models refuse a forced tool call (Anthropic's thinking models: "tool_choice ... not supported for this + // model"). Forcing one only nudges a where-is question toward evidence, so ask again without it. + requestOptions = ctx.ChatOptions; + } catch (Exception ex) when (IsTransientStreamError(ex) && streamAttempt < limits.MaxStreamRetries) { // Transient stream disconnection — retry automatically with back-off. diff --git a/tests/FuseraftCli.Tests/ReplCommandTests.cs b/tests/FuseraftCli.Tests/ReplCommandTests.cs index e4e47844..039c3da2 100644 --- a/tests/FuseraftCli.Tests/ReplCommandTests.cs +++ b/tests/FuseraftCli.Tests/ReplCommandTests.cs @@ -1,6 +1,7 @@ using System.Text.Json; using fuseraft.Cli.Commands.Repl; using fuseraft.Cli.Protocol; +using fuseraft.Core.Models.Config; namespace FuseraftCli.Tests; @@ -22,4 +23,34 @@ public void ReadyEvent_KeepsEveryFieldVsCodeReads_AndAddsProtocolVersion() Assert.Contains("/title", root.GetProperty("midTurnCommands").EnumerateArray().Select(e => e.GetString())); Assert.Equal(WireProtocol.Version, root.GetProperty("protocolVersion").GetInt32()); } + + [Fact] + public void HostModel_CompletesAConfigThatHasSettingsButNoModel() + { + var saved = new UserConfig { Repl = new() { Bell = false } }; + + var cfg = ReplCommand.WithHostModel(saved, " claude-x ", "anthropic"); + + Assert.Same(saved, cfg); + Assert.Equal("claude-x", cfg!.ModelId); + Assert.Equal("anthropic", cfg.Provider); + } + + [Fact] + public void HostModel_LeavesAConfiguredModelAlone() + { + var saved = new UserConfig { ModelId = "gpt-x", Provider = "openai" }; + + var cfg = ReplCommand.WithHostModel(saved, "claude-x", "anthropic"); + + Assert.Equal("gpt-x", cfg!.ModelId); + Assert.Equal("openai", cfg.Provider); + } + + [Fact] + public void HostModel_MakesAConfigWhenThereIsNone_AndNothingWithoutAModel() + { + Assert.Equal("claude-x", ReplCommand.WithHostModel(null, "claude-x", null)!.ModelId); + Assert.Null(ReplCommand.WithHostModel(null, null, "anthropic")); + } } diff --git a/tests/FuseraftCli.Tests/ReplLimitsTurnTests.cs b/tests/FuseraftCli.Tests/ReplLimitsTurnTests.cs index 7c664b19..578a854d 100644 --- a/tests/FuseraftCli.Tests/ReplLimitsTurnTests.cs +++ b/tests/FuseraftCli.Tests/ReplLimitsTurnTests.cs @@ -354,6 +354,114 @@ public void StopNote_JoinsAClosingTextMessage_RatherThanAddingASecondAssistantMe Assert.Contains("input-token budget", history[^1].Text); } + // ── forced tool choice ─────────────────────────────────────────────────── + + // Refuses any request that forces a tool call, as Anthropic's thinking models do. + private sealed class RefusesForcedToolChoice : IChatClient + { + public readonly List Modes = []; + + public ChatClientMetadata Metadata => new("test", null!, "stub"); + + public Task GetResponseAsync( + IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + => Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, string.Empty))); + + public IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + Modes.Add(options?.ToolMode); + if (options?.ToolMode is RequiredChatToolMode) + throw new InvalidOperationException( + "{\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"tool_choice: type \\\"tool\\\" and \\\"any\\\" are not supported for this model.\"}}"); + return Answer("It's in src/config.cs."); + } + + public object? GetService(Type serviceType, object? key = null) => null; + public void Dispose() { } + } + + [Fact] + public async Task AWhereIsQuestion_IsAskedAgainWithoutForcingATool_WhenTheModelRefusesForcing() + { + var client = new RefusesForcedToolChoice(); + var (ctx, _) = NewContext(client, repl: null); + ctx.ChatOptions = new ChatOptions { Tools = [AIFunctionFactory.Create(() => "x", "read_file")] }; + + await ReplTurn.ExecuteAsync(ctx, "where is the config loaded?", isStepRequest: false, capturePlan: false, activeStep: null, CancellationToken.None); + + Assert.Equal(2, client.Modes.Count); + Assert.IsType(client.Modes[0]); + Assert.IsNotType(client.Modes[1]); + Assert.Equal("It's in src/config.cs.", ctx.History[^1].Text); + } + + [Fact] + public void OnlyAToolChoiceError_CountsAsARefusalOfForcing() + { + Assert.True(ReplTurn.IsToolChoiceRejection(new InvalidOperationException("tool_choice: type \"any\" is not supported for this model."))); + Assert.False(ReplTurn.IsToolChoiceRejection(new InvalidOperationException("prompt is too long"))); + } + + // ── dropped replies ────────────────────────────────────────────────────── + + // A write the tool refused, then a reply that's only the tool's error line, which the REPL drops as leaked + // tool text. + private static async IAsyncEnumerable RefusedWriteThenUnusableReply(string callId) + { + yield return new ChatResponseUpdate + { + Role = ChatRole.Assistant, + Contents = [new FunctionCallContent(callId, "write_file", new Dictionary { ["path"] = "a.md" })], + }; + await Task.Yield(); + yield return new ChatResponseUpdate { Role = ChatRole.Tool, Contents = [new FunctionResultContent(callId, "[ERROR] WRITE BLOCKED")] }; + yield return new ChatResponseUpdate { Role = ChatRole.Assistant, Contents = [new TextContent("[ERROR] WRITE BLOCKED")] }; + yield return Usage(10); + } + + [Fact] + public async Task DroppedReply_KeepsTheToolWork_SoTheRetryKnowsItRan() + { + var client = new ScriptedClient(n => n == 0 ? RefusedWriteThenUnusableReply("c0") : Answer("The write was refused.")); + var (ctx, _) = NewContext(client, repl: null); + + await RunAsync(ctx); + + Assert.Equal(2, client.Streams); + Assert.Equal(1, ResultCount(client.Requests[1], "c0")); + Assert.DoesNotContain(client.Requests[1], m => m.Role == ChatRole.Assistant && m.Text.StartsWith("[ERROR]", StringComparison.Ordinal)); + Assert.Equal(1, ResultCount(ctx.History, "c0")); + Assert.Equal("The write was refused.", ctx.History[^1].Text); + } + + [Fact] + public async Task ReplyDroppedTwice_KeepsBothAttemptsToolWork() + { + var client = new ScriptedClient(n => RefusedWriteThenUnusableReply($"c{n}")); + var (ctx, _) = NewContext(client, repl: null); + + await RunAsync(ctx); + + Assert.Equal(2, client.Streams); + Assert.Equal(1, ResultCount(ctx.History, "c0")); + Assert.Equal(1, ResultCount(ctx.History, "c1")); + Assert.StartsWith(ReplTurn.StoppedEarlyNotePrefix, ctx.History[^1].Text); + } + + [Fact] + public async Task EmptyReplyWithNoTools_AddsNothingBeforeTheRetry() + { + var client = new ScriptedClient(n => n == 0 ? Answer("") : Answer("Hello.")); + var (ctx, _) = NewContext(client, repl: null); + + await RunAsync(ctx); + + Assert.Equal(2, client.Streams); + Assert.DoesNotContain(client.Requests[1], m => m.Text.StartsWith(ReplTurn.StoppedEarlyNotePrefix, StringComparison.Ordinal)); + Assert.Equal("Hello.", ctx.History[^1].Text); + } + // Previously the failure cutoff on a turn with no text fell into the empty-reply retry: the // stopped turn's tool calls were dropped and a whole extra turn ran. Now the tool calls are // kept, and an unattended turn (JSON mode here) gets one reply without tools, not a retry. diff --git a/tests/FuseraftCli.Tests/ReplTurnSanitizeAssistantResponseTests.cs b/tests/FuseraftCli.Tests/ReplTurnSanitizeAssistantResponseTests.cs index 807cd985..ef2065b7 100644 --- a/tests/FuseraftCli.Tests/ReplTurnSanitizeAssistantResponseTests.cs +++ b/tests/FuseraftCli.Tests/ReplTurnSanitizeAssistantResponseTests.cs @@ -97,4 +97,32 @@ public void EmptyResponse_ReturnsEmptyWithNoWarning() Assert.Equal(string.Empty, result); Assert.Null(warning); } + + [Fact] + public void AnAnswerQuotingAToolErrorInACodeFence_IsKept() + { + // From a live session: the model reported a refused write by quoting the error, and the answer was dropped. + const string answer = + "The tool refused the write. Its exact error message was:\n\n```\n" + + "[ERROR] WRITE BLOCKED — '/repo/config.txt' changed after you last read or wrote it, so content built from " + + "your copy would undo that change. 1. Re-read it with read_file (startLine/maxLines for the part you need). " + + "2. Then make the edit with patch_file, or write_file the full updated content.\n```\n\n" + + "I stopped there and didn't retry. `config.txt` is unchanged."; + + var result = ReplTurn.SanitizeAssistantResponse(answer, out var warning); + + Assert.Equal(answer, result); + Assert.Null(warning); + } + + [Fact] + public void ResultLinesOutsideAFence_AreStillDropped_WhenAFencedBlockIsNearby() + { + const string leaked = "[OK] Written 120 chars to foo.cs\n[OK] Patched 'bar.cs': replaced 2-line block\n```\nx\n```"; + + var result = ReplTurn.SanitizeAssistantResponse(leaked, out var warning); + + Assert.Equal(string.Empty, result); + Assert.NotNull(warning); + } }