Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .changeset/tidy-tools-continue.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
---
"@voltagent/core": patch
---

Prevent model retries and fallback from replaying tool calls in `generateText`, including when
an approved tool executes before the first model step. Surface the original failure to the caller.

Fixes #1537.
38 changes: 35 additions & 3 deletions packages/core/src/agent/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1347,10 +1347,33 @@ export class Agent {
| undefined;
applyForcedToolChoice(aiSDKOptions, forcedToolChoice);

let hasToolExecution = false;
const trackedTools = Object.fromEntries(
Object.entries<AITool>(tools).map(([name, tool]) => {
const execute = tool.execute;
if (!execute) {
return [name, tool];
}
return [
name,
{
...tool,
execute: (...args: Parameters<typeof execute>) => {
// Approved tools can run before the SDK starts its first step.
hasToolExecution = true;
return execute.call(tool, ...args);
},
},
];
}),
);
const handleStep = this.createStepHandler(oc, options);
const { result, modelName: effectiveModelName } = await this.executeWithModelFallback({
oc,
operation: "generateText",
options,
// Restarting generateText from the original messages can replay tool side effects.
canReplay: () => !hasToolExecution,
run: async ({
model: resolvedModel,
modelName: resolvedModelName,
Expand Down Expand Up @@ -1386,7 +1409,7 @@ export class Agent {
generateText({
model: resolvedModel,
messages,
tools,
tools: trackedTools,
// Default values
temperature: this.temperature,
maxOutputTokens: this.maxOutputTokens,
Expand All @@ -1400,7 +1423,14 @@ export class Agent {
providerOptions,
// VoltAgent controlled (these should not be overridden)
abortSignal: oc.abortController.signal,
onStepFinish: this.createStepHandler(oc, options),
onStepFinish: async (step) => {
hasToolExecution ||= Boolean(
step.content?.some(
(part) => part.type === "tool-result" || part.type === "tool-error",
),
);
await handleStep(step);
},
}),
);

Expand Down Expand Up @@ -5928,11 +5958,13 @@ export class Agent {
oc,
operation,
options,
canReplay,
run,
}: {
oc: OperationContext;
operation: LLMOperation;
options?: BaseGenerationOptions;
canReplay?: () => boolean;
run: (args: {
model: LanguageModel;
modelName: string;
Expand Down Expand Up @@ -6063,7 +6095,7 @@ export class Agent {
return { result, modelName, modelIndex: index, maxRetries };
} catch (error) {
lastError = error;
if (oc.abortController.signal.aborted) {
if (oc.abortController.signal.aborted || canReplay?.() === false) {
throw error;
}
const fallbackEligible = this.shouldFallbackOnError(error);
Expand Down
98 changes: 98 additions & 0 deletions packages/core/src/agent/generate-text-replay.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
import { APICallError, type UIMessage } from "ai";
import { MockLanguageModelV3 } from "ai/test";
import { describe, expect, it, vi } from "vitest";
import { z } from "zod";
import { createTool } from "../tool";
import { Agent } from "./agent";

const usage = {
inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 },
outputTokens: { total: 5, text: 5, reasoning: 0 },
};

describe.each([
{ scenario: "a tool called by the model", resumeApproval: false },
{ scenario: "a tool resumed from approval", resumeApproval: true },
])("generateText after $scenario", ({ resumeApproval }) => {
it.each(["retry", "fallback"] as const)("does not replay a tool on model %s", async (mode) => {
const providerError = new APICallError({
message: "Model overloaded",
url: "https://example.com/model",
requestBodyValues: {},
statusCode: 503,
isRetryable: true,
});
const execute = vi.fn(async () => "recorded");
let hasFailed = false;
const primary = new MockLanguageModelV3({
doGenerate: async ({ prompt }) => {
if (!prompt.some((message) => message.role === "tool")) {
return {
content: [{ type: "tool-call", toolCallId: "call-1", toolName: "record", input: "{}" }],
finishReason: { unified: "tool-calls", raw: undefined },
usage,
warnings: [],
};
}
if (!hasFailed) {
hasFailed = true;
throw providerError;
}
return {
content: [{ type: "text", text: "Done" }],
finishReason: { unified: "stop", raw: undefined },
usage,
warnings: [],
};
},
});
const generateFallback = vi.fn(async () => {
throw new Error("The fallback model should never be called");
});
const fallback = new MockLanguageModelV3({ doGenerate: generateFallback });
const agent = new Agent({
name: "tool-replay",
instructions: "Record the action",
model: mode === "retry" ? primary : [{ model: primary, maxRetries: 0 }, { model: fallback }],
maxRetries: mode === "retry" ? 1 : 0,
maxSteps: 2,
memory: false,
tools: [
createTool({
name: "record",
description: "Record an action",
parameters: z.object({}),
needsApproval: resumeApproval,
execute,
}),
],
});

const approvedMessages: UIMessage[] = [
{
id: "user-message",
role: "user",
parts: [{ type: "text", text: "Record the action" }],
},
{
id: "assistant-message",
role: "assistant",
parts: [
{
type: "tool-record",
state: "approval-responded",
toolCallId: "call-1",
input: {},
approval: { id: "approval-1", approved: true },
},
],
},
];
const result = agent.generateText(resumeApproval ? approvedMessages : "Record the action");

await expect(result).rejects.toBe(providerError);
expect(execute).toHaveBeenCalledTimes(1);
expect(primary.doGenerateCalls).toHaveLength(resumeApproval ? 1 : 2);
expect(generateFallback).not.toHaveBeenCalled();
});
});
5 changes: 5 additions & 0 deletions website/docs/agents/retries-fallback.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,11 @@ Fallback **does not** trigger for:
- Guardrail blocks.
- Tool execution errors.

For `generateText`, once a local tool starts or a step records a tool result or error, a
subsequent failure is returned to the caller. This includes approved tools that run before
the first model step. Model retries and fallback would restart generation from the original
messages and could repeat tool side effects.

Streaming behavior:

- Retries and fallback happen only if the stream fails before the first output chunk.
Expand Down