Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions src/Cli/Commands/Repl/ReplCommand.cs
Original file line number Diff line number Diff line change
Expand Up @@ -260,8 +260,7 @@ private async Task<int> RunAsync(
{
// A host that names the model (fuseraft-web, per account) needs no config file: --model plus
// FUSERAFT_API_KEY is a complete setup. Nothing here is saved unless --save asks.
if (userCfg is null && !string.IsNullOrWhiteSpace(settings.Model))
userCfg = new UserConfig { ModelId = settings.Model.Trim(), Provider = settings.Provider?.Trim() ?? string.Empty };
userCfg = WithHostModel(userCfg, settings.Model, settings.Provider);

// Running from VS Code. Prefer an API key explicitly injected by the
// extension (FUSERAFT_API_KEY), then fall back to any legacy plaintext
Expand Down Expand Up @@ -1015,6 +1014,23 @@ private async Task<int> RunAsync(
return null;
}

/// <summary>
/// The config a host's --model completes: a new one when there's no config file, or the file's when it has settings
/// but no model yet, as after <c>fuseraft settings set</c> or the VS Code settings panel on a fresh install.
/// </summary>
internal static UserConfig? WithHostModel(UserConfig? userCfg, string? model, string? provider)
{
if (string.IsNullOrWhiteSpace(model)) return userCfg;
if (userCfg is null)
return new UserConfig { ModelId = model.Trim(), Provider = provider?.Trim() ?? string.Empty };
if (string.IsNullOrWhiteSpace(userCfg.ModelId))
{
userCfg.ModelId = model.Trim();
if (!string.IsNullOrWhiteSpace(provider)) userCfg.Provider = provider.Trim();
}
return userCfg;
}

private static string? ResolveModelId(ReplSettings settings, UserConfig? userCfg)
{
var modelId = settings.Model?.Trim();
Expand Down
33 changes: 31 additions & 2 deletions src/Cli/Commands/Repl/ReplTurn.cs
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,11 @@ internal static void AppendStoppedEarlyNote(List<ChatMessage> history, string re
return forced;
}

// A provider's refusal of the forced tool choice BuildRequestOptions asked for, rather than of the request as a whole.
internal static bool IsToolChoiceRejection(Exception ex) =>
ex is not OperationCanceledException &&
$"{ex.Message} {ex.InnerException?.Message}".Contains("tool_choice", StringComparison.OrdinalIgnoreCase);

// The options for a reply that mustn't call tools. The tools stay declared, since a provider may reject a
// history with tool calls in it when none are, and the model is told not to use them.
internal static ChatOptions? WithoutToolCalls(ChatOptions? baseOptions)
Expand Down Expand Up @@ -898,6 +903,17 @@ await ctx.Emitter.EmitAsync(EventTypes.UserInput, turn: ctx.TurnIndex,
responseText = SanitizeAssistantResponse(responseText, out var warningMessage);
if (!capturePlan && responseText.Length == 0 && !stoppedEarly)
{
// The reply is dropped, but the tools it called have run. Keep their calls and results, so the retry below
// and later turns know what was done: told nothing, the model ran a refused write again, and would rerun
// a commit or a deploy the same way.
var kept = CompletedRounds(rawUpdates);
if (kept.Count > 0)
{
ctx.History.AddRange(kept);
RepairDanglingToolCalls(ctx.History);
AppendStoppedEarlyNote(ctx.History, "the reply after these tool calls was empty or not a user-facing answer, so it was dropped");
}

// Gated on its own emptyResponseRetried flag rather than the shared isCorrectionTurn
// flag — an empty response can legitimately occur mid-way through an in-progress
// todo-correction chain (a nudge message can provoke a bare tool call with no
Expand Down Expand Up @@ -1350,20 +1366,27 @@ internal static string SanitizeAssistantResponse(string responseText, out string
// response's non-blank content, by character count rather than raw line count — a leaked
// single tool result is often accompanied by a short structural wrapper (e.g. a "Tool
// Calls]" header or a bare tool name line) that would otherwise dilute a naive per-line
// majority check below the threshold that actually caught the real #119 incident.
// majority check below the threshold that actually caught the real #119 incident. Lines in a code
// fence don't count as leaked: a model quoting a tool's error to the user puts it in one.
private static bool IsDominatedByToolResultLines(string trimmed)
{
var totalChars = 0;
var matchedChars = 0;
var hasMatch = false;
var inFence = false;

foreach (var rawLine in trimmed.Split('\n'))
{
var line = rawLine.Trim();
if (line.Length == 0) continue;

totalChars += line.Length;
if (ToolResultLinePattern.IsMatch(line))
if (line.StartsWith("```", StringComparison.Ordinal) || line.StartsWith("~~~", StringComparison.Ordinal))
{
inFence = !inFence;
continue;
}
if (!inFence && ToolResultLinePattern.IsMatch(line))
{
hasMatch = true;
matchedChars += line.Length;
Expand Down Expand Up @@ -2051,6 +2074,12 @@ await KeepGoingAfterFailuresAsync(ctx, consecutiveToolFailures, lastToolFailureD
ctx.ActiveCts = null;
return TurnStreamResult.MakeFailed(toolCallsThisTurn, limits.MaxIdenticalToolCalls);
}
catch (Exception ex) when (requestOptions?.ToolMode is RequiredChatToolMode && rawUpdates.Count == 0 && IsToolChoiceRejection(ex))
{
// Some models refuse a forced tool call (Anthropic's thinking models: "tool_choice ... not supported for this
// model"). Forcing one only nudges a where-is question toward evidence, so ask again without it.
requestOptions = ctx.ChatOptions;
}
catch (Exception ex) when (IsTransientStreamError(ex) && streamAttempt < limits.MaxStreamRetries)
{
// Transient stream disconnection — retry automatically with back-off.
Expand Down
31 changes: 31 additions & 0 deletions tests/FuseraftCli.Tests/ReplCommandTests.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System.Text.Json;
using fuseraft.Cli.Commands.Repl;
using fuseraft.Cli.Protocol;
using fuseraft.Core.Models.Config;

namespace FuseraftCli.Tests;

Expand All @@ -22,4 +23,34 @@ public void ReadyEvent_KeepsEveryFieldVsCodeReads_AndAddsProtocolVersion()
Assert.Contains("/title", root.GetProperty("midTurnCommands").EnumerateArray().Select(e => e.GetString()));
Assert.Equal(WireProtocol.Version, root.GetProperty("protocolVersion").GetInt32());
}

[Fact]
public void HostModel_CompletesAConfigThatHasSettingsButNoModel()
{
var saved = new UserConfig { Repl = new() { Bell = false } };

var cfg = ReplCommand.WithHostModel(saved, " claude-x ", "anthropic");

Assert.Same(saved, cfg);
Assert.Equal("claude-x", cfg!.ModelId);
Assert.Equal("anthropic", cfg.Provider);
}

[Fact]
public void HostModel_LeavesAConfiguredModelAlone()
{
var saved = new UserConfig { ModelId = "gpt-x", Provider = "openai" };

var cfg = ReplCommand.WithHostModel(saved, "claude-x", "anthropic");

Assert.Equal("gpt-x", cfg!.ModelId);
Assert.Equal("openai", cfg.Provider);
}

[Fact]
public void HostModel_MakesAConfigWhenThereIsNone_AndNothingWithoutAModel()
{
Assert.Equal("claude-x", ReplCommand.WithHostModel(null, "claude-x", null)!.ModelId);
Assert.Null(ReplCommand.WithHostModel(null, null, "anthropic"));
}
}
108 changes: 108 additions & 0 deletions tests/FuseraftCli.Tests/ReplLimitsTurnTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,114 @@ public void StopNote_JoinsAClosingTextMessage_RatherThanAddingASecondAssistantMe
Assert.Contains("input-token budget", history[^1].Text);
}

// ── forced tool choice ───────────────────────────────────────────────────

// Refuses any request that forces a tool call, as Anthropic's thinking models do.
private sealed class RefusesForcedToolChoice : IChatClient
{
public readonly List<ChatToolMode?> Modes = [];

public ChatClientMetadata Metadata => new("test", null!, "stub");

public Task<ChatResponse> GetResponseAsync(
IEnumerable<ChatMessage> messages, ChatOptions? options = null, CancellationToken cancellationToken = default)
=> Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, string.Empty)));

public IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
IEnumerable<ChatMessage> messages, ChatOptions? options = null, CancellationToken cancellationToken = default)
{
Modes.Add(options?.ToolMode);
if (options?.ToolMode is RequiredChatToolMode)
throw new InvalidOperationException(
"{\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"tool_choice: type \\\"tool\\\" and \\\"any\\\" are not supported for this model.\"}}");
return Answer("It's in src/config.cs.");
}

public object? GetService(Type serviceType, object? key = null) => null;
public void Dispose() { }
}

[Fact]
public async Task AWhereIsQuestion_IsAskedAgainWithoutForcingATool_WhenTheModelRefusesForcing()
{
var client = new RefusesForcedToolChoice();
var (ctx, _) = NewContext(client, repl: null);
ctx.ChatOptions = new ChatOptions { Tools = [AIFunctionFactory.Create(() => "x", "read_file")] };

await ReplTurn.ExecuteAsync(ctx, "where is the config loaded?", isStepRequest: false, capturePlan: false, activeStep: null, CancellationToken.None);

Assert.Equal(2, client.Modes.Count);
Assert.IsType<RequiredChatToolMode>(client.Modes[0]);
Assert.IsNotType<RequiredChatToolMode>(client.Modes[1]);
Assert.Equal("It's in src/config.cs.", ctx.History[^1].Text);
}

[Fact]
public void OnlyAToolChoiceError_CountsAsARefusalOfForcing()
{
Assert.True(ReplTurn.IsToolChoiceRejection(new InvalidOperationException("tool_choice: type \"any\" is not supported for this model.")));
Assert.False(ReplTurn.IsToolChoiceRejection(new InvalidOperationException("prompt is too long")));
}

// ── dropped replies ──────────────────────────────────────────────────────

// A write the tool refused, then a reply that's only the tool's error line, which the REPL drops as leaked
// tool text.
private static async IAsyncEnumerable<ChatResponseUpdate> RefusedWriteThenUnusableReply(string callId)
{
yield return new ChatResponseUpdate
{
Role = ChatRole.Assistant,
Contents = [new FunctionCallContent(callId, "write_file", new Dictionary<string, object?> { ["path"] = "a.md" })],
};
await Task.Yield();
yield return new ChatResponseUpdate { Role = ChatRole.Tool, Contents = [new FunctionResultContent(callId, "[ERROR] WRITE BLOCKED")] };
yield return new ChatResponseUpdate { Role = ChatRole.Assistant, Contents = [new TextContent("[ERROR] WRITE BLOCKED")] };
yield return Usage(10);
}

[Fact]
public async Task DroppedReply_KeepsTheToolWork_SoTheRetryKnowsItRan()
{
var client = new ScriptedClient(n => n == 0 ? RefusedWriteThenUnusableReply("c0") : Answer("The write was refused."));
var (ctx, _) = NewContext(client, repl: null);

await RunAsync(ctx);

Assert.Equal(2, client.Streams);
Assert.Equal(1, ResultCount(client.Requests[1], "c0"));
Assert.DoesNotContain(client.Requests[1], m => m.Role == ChatRole.Assistant && m.Text.StartsWith("[ERROR]", StringComparison.Ordinal));
Assert.Equal(1, ResultCount(ctx.History, "c0"));
Assert.Equal("The write was refused.", ctx.History[^1].Text);
}

[Fact]
public async Task ReplyDroppedTwice_KeepsBothAttemptsToolWork()
{
var client = new ScriptedClient(n => RefusedWriteThenUnusableReply($"c{n}"));
var (ctx, _) = NewContext(client, repl: null);

await RunAsync(ctx);

Assert.Equal(2, client.Streams);
Assert.Equal(1, ResultCount(ctx.History, "c0"));
Assert.Equal(1, ResultCount(ctx.History, "c1"));
Assert.StartsWith(ReplTurn.StoppedEarlyNotePrefix, ctx.History[^1].Text);
}

[Fact]
public async Task EmptyReplyWithNoTools_AddsNothingBeforeTheRetry()
{
var client = new ScriptedClient(n => n == 0 ? Answer("") : Answer("Hello."));
var (ctx, _) = NewContext(client, repl: null);

await RunAsync(ctx);

Assert.Equal(2, client.Streams);
Assert.DoesNotContain(client.Requests[1], m => m.Text.StartsWith(ReplTurn.StoppedEarlyNotePrefix, StringComparison.Ordinal));
Assert.Equal("Hello.", ctx.History[^1].Text);
}

// Previously the failure cutoff on a turn with no text fell into the empty-reply retry: the
// stopped turn's tool calls were dropped and a whole extra turn ran. Now the tool calls are
// kept, and an unattended turn (JSON mode here) gets one reply without tools, not a retry.
Expand Down
28 changes: 28 additions & 0 deletions tests/FuseraftCli.Tests/ReplTurnSanitizeAssistantResponseTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -97,4 +97,32 @@ public void EmptyResponse_ReturnsEmptyWithNoWarning()
Assert.Equal(string.Empty, result);
Assert.Null(warning);
}

[Fact]
public void AnAnswerQuotingAToolErrorInACodeFence_IsKept()
{
// From a live session: the model reported a refused write by quoting the error, and the answer was dropped.
const string answer =
"The tool refused the write. Its exact error message was:\n\n```\n" +
"[ERROR] WRITE BLOCKED — '/repo/config.txt' changed after you last read or wrote it, so content built from " +
"your copy would undo that change. 1. Re-read it with read_file (startLine/maxLines for the part you need). " +
"2. Then make the edit with patch_file, or write_file the full updated content.\n```\n\n" +
"I stopped there and didn't retry. `config.txt` is unchanged.";

var result = ReplTurn.SanitizeAssistantResponse(answer, out var warning);

Assert.Equal(answer, result);
Assert.Null(warning);
}

[Fact]
public void ResultLinesOutsideAFence_AreStillDropped_WhenAFencedBlockIsNearby()
{
const string leaked = "[OK] Written 120 chars to foo.cs\n[OK] Patched 'bar.cs': replaced 2-line block\n```\nx\n```";

var result = ReplTurn.SanitizeAssistantResponse(leaked, out var warning);

Assert.Equal(string.Empty, result);
Assert.NotNull(warning);
}
}
Loading