Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,10 @@ export class MessagesWireApiProvider implements ProviderAdapter {
}

createStreamDecoder(): StreamDecoder {
// message_start carries the input/cache counts; message_delta carries
// the cumulative output count. Report one combined usage at the end so
// callers that sum usage per piece don't count the input twice.
let startUsage: Message["usage"] | undefined;
return {
push: (raw: string): StreamPiece => {
const evt = JSON.parse(raw) as RawMessageStreamEvent;
Expand All @@ -208,13 +212,27 @@ export class MessagesWireApiProvider implements ProviderAdapter {
piece.text = (evt.delta as TextDelta).text;
}

if (evt.type === "message_delta") {
if (evt.type === "message_start") {
startUsage = evt.message.usage;
} else if (evt.type === "message_delta") {
if (evt.delta.stop_reason === "max_tokens") {
throw new Error(incompleteOutputMessage);
}
piece.usage = usageFromMessagesUsage(evt.usage);
} else if (evt.type === "message_start") {
piece.usage = usageFromMessagesUsage(evt.message.usage);
piece.usage = usageFromMessagesUsage({
input_tokens:
evt.usage.input_tokens ??
startUsage?.input_tokens ??
null,
output_tokens: evt.usage.output_tokens,
cache_creation_input_tokens:
evt.usage.cache_creation_input_tokens ??
startUsage?.cache_creation_input_tokens ??
null,
cache_read_input_tokens:
evt.usage.cache_read_input_tokens ??
startUsage?.cache_read_input_tokens ??
null,
});
}

return piece;
Expand Down
37 changes: 37 additions & 0 deletions ts/packages/aiclient/test/providerAdapter.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -475,6 +475,43 @@ describe("MessagesWireApiProvider", () => {
);
expect(piece.text).toBe("hi");
});

test("streaming decoder reports usage once, combining message_start input with message_delta output", () => {
const decoder = adapter.createStreamDecoder!(makeRequest());
const start = decoder.push(
JSON.stringify({
type: "message_start",
message: {
content: [],
usage: {
input_tokens: 10,
output_tokens: 1,
cache_creation_input_tokens: 20,
cache_read_input_tokens: 30,
},
},
}),
);
const delta = decoder.push(
JSON.stringify({
type: "message_delta",
delta: { stop_reason: "end_turn", stop_sequence: null },
usage: {
input_tokens: null,
output_tokens: 42,
cache_creation_input_tokens: null,
cache_read_input_tokens: null,
},
}),
);
expect(start.usage).toBeUndefined();
expect(delta.usage).toEqual({
prompt_tokens: 60,
completion_tokens: 42,
total_tokens: 102,
cached_tokens: 30,
});
});
});

describe("ResponsesWireApiProvider", () => {
Expand Down
Loading