Repository navigation
fix(core): persist approved tool results immediately after execution #1541
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from 2 commits
ff2da6b
359b9f4
1577ff6
107e066
0f7eabe
9183914
6724ebf
7eb35be
ca11774
44c59d8
9f9b2ff
2281c53
09fa531
1c0e2b1
fbbdf3f
f81a3ef
7103a38
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| --- | ||
| "@voltagent/core": patch | ||
| --- | ||
|
|
||
| fix(core): persist client approval decisions onto buffered tool parts so restored conversations keep `approval-responded` state instead of re-validating stale parts and throwing `AI_TypeValidationError` |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| --- | ||
| "@voltagent/core": patch | ||
| --- | ||
|
|
||
| fix(core): flush approved `needsApproval` tool results to memory as soon as execution completes and preserve executed tool parts when a client resends an older message, so an interrupted turn no longer causes a resent approval to re-execute the tool without a new approval prompt |
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -4296,6 +4296,72 @@ export class Agent { | |||||
| oc.output = undefined; | ||||||
| } | ||||||
|
|
||||||
| /** | ||||||
| * Persists the result of an approved `needsApproval` tool as soon as its | ||||||
| * execution completes. | ||||||
| * | ||||||
| * The AI SDK executes approved tool calls before step 1, but their results | ||||||
| * normally reach memory only via step 1's `onStepFinish` - after the next | ||||||
| * model call. If the turn is interrupted in between (client disconnect, | ||||||
| * crash, provider error), memory keeps `approval-responded` with no output | ||||||
| * and a resent approval re-executes the tool without a new approval prompt. | ||||||
| * Flushing here narrows that window; it is not exactly-once semantics - a | ||||||
| * crash between the tool's own commit and this flush can still require | ||||||
| * tool-side idempotency. See https://github.com/VoltAgent/voltagent/issues/1538 | ||||||
| */ | ||||||
| private async persistApprovedToolResult(params: { | ||||||
| tool: BaseTool; | ||||||
| toolCallId: string; | ||||||
| output: unknown; | ||||||
| oc: OperationContext; | ||||||
| }): Promise<void> { | ||||||
| const { tool, toolCallId, output, oc } = params; | ||||||
| try { | ||||||
| // The SDK only invokes `execute` for a `needsApproval` tool after an | ||||||
| // explicit client approval, so reaching this point means the call was | ||||||
| // approved. Nothing is inferred from outputs or message state. | ||||||
| if (!tool.needsApproval) return; | ||||||
| if (!this.shouldPersistMemoryForContext(oc)) return; | ||||||
|
|
||||||
| const conversationPersistence = this.getConversationPersistenceOptionsForContext(oc); | ||||||
| if (conversationPersistence.mode !== "step" || !conversationPersistence.flushOnToolResult) { | ||||||
| return; | ||||||
| } | ||||||
|
|
||||||
| const persistQueue = this.getMemoryPersistQueue(oc); | ||||||
| if (!persistQueue) return; | ||||||
|
|
||||||
| // Mirror the AI SDK's model-output wrapping (createToolModelOutput) so | ||||||
| // the early flush stores the same shape the step flush would. | ||||||
| const modelOutput = tool.toModelOutput | ||||||
| ? await tool.toModelOutput({ output: output as never }) | ||||||
| : typeof output === "string" | ||||||
| ? { type: "text" as const, value: output } | ||||||
| : { type: "json" as const, value: (output === undefined ? null : output) as any }; | ||||||
|
|
||||||
| const buffer = this.getConversationBuffer(oc); | ||||||
| buffer.addModelMessages( | ||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Seed the approved call before merging its result. With Seed the incoming approved tool part, including its input and approval, before merging the result. Cover an interruption before
|
||||||
| [ | ||||||
| { | ||||||
| role: "tool", | ||||||
| content: [ | ||||||
| { type: "tool-result", toolCallId, toolName: tool.name, output: modelOutput }, | ||||||
| ], | ||||||
| } as ModelMessage, | ||||||
| ], | ||||||
| "response", | ||||||
| ); | ||||||
| await persistQueue.flush(buffer, oc); | ||||||
| } catch (error) { | ||||||
| oc.logger.debug("Failed to persist approved tool result", { | ||||||
| error, | ||||||
| toolCallId, | ||||||
| conversationId: oc.conversationId, | ||||||
| userId: oc.userId, | ||||||
| }); | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| private getConversationBuffer(oc: OperationContext): ConversationBuffer { | ||||||
| let buffer = oc.systemContext.get(BUFFER_CONTEXT_KEY) as ConversationBuffer | undefined; | ||||||
| if (!buffer) { | ||||||
|
|
@@ -5075,6 +5141,11 @@ export class Agent { | |||||
| }, | ||||||
| ); | ||||||
| buffer.ingestUIMessages(memMessages, true); | ||||||
| // Apply approval decisions carried by the input so buffered | ||||||
| // approval-requested parts don't keep a stale `approval`. | ||||||
| if (Array.isArray(resolvedInput) && (resolvedInput as any[])[0]?.parts) { | ||||||
| buffer.applyApprovalResponses(resolvedInput as UIMessage[]); | ||||||
| } | ||||||
| return memMessages; | ||||||
| } | ||||||
| // Regular memory context | ||||||
|
|
@@ -5103,6 +5174,14 @@ export class Agent { | |||||
|
|
||||||
| buffer.ingestUIMessages(result.messages, true); | ||||||
|
|
||||||
| // Apply approval decisions carried by the input so buffered | ||||||
| // approval-requested parts don't keep a stale `approval` | ||||||
| // (otherwise a later `output-available` part fails AI SDK | ||||||
| // validation and breaks every subsequent turn). | ||||||
| if (Array.isArray(inputForMemory)) { | ||||||
| buffer.applyApprovalResponses(inputForMemory); | ||||||
| } | ||||||
|
|
||||||
| if ( | ||||||
| shouldPersistMemory && | ||||||
| speculativeInputGuardrail && | ||||||
|
|
@@ -5244,6 +5323,53 @@ export class Agent { | |||||
| .filter((id): id is string => typeof id === "string" && id.trim().length > 0), | ||||||
| ); | ||||||
|
|
||||||
| // A stored tool part in a terminal output state is evidence the tool | ||||||
| // already executed. When the client resends an older version of the | ||||||
| // same message (e.g. an approval retry after an interrupted turn), | ||||||
| // keep the executed part so the SDK does not re-run the tool without | ||||||
| // a new approval prompt. See | ||||||
| // https://github.com/VoltAgent/voltagent/issues/1538 | ||||||
| const terminalToolStates = new Set(["output-available", "output-error", "output-denied"]); | ||||||
| const storedTerminalParts = new Map<string, Map<string, any>>(); | ||||||
| for (const message of messages) { | ||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P1: This lookup misses executed results once the approval message falls outside Prompt for AI agents |
||||||
| if (!idsToReplace.has(message.id)) continue; | ||||||
| for (const part of (message.parts ?? []) as any[]) { | ||||||
| if ( | ||||||
| typeof part?.type === "string" && | ||||||
| part.type.startsWith("tool-") && | ||||||
| part.toolCallId && | ||||||
| terminalToolStates.has(part.state) | ||||||
| ) { | ||||||
| let byToolCallId = storedTerminalParts.get(message.id); | ||||||
| if (!byToolCallId) { | ||||||
| byToolCallId = new Map(); | ||||||
| storedTerminalParts.set(message.id, byToolCallId); | ||||||
| } | ||||||
| byToolCallId.set(part.toolCallId, part); | ||||||
| } | ||||||
| } | ||||||
| } | ||||||
| if (storedTerminalParts.size > 0) { | ||||||
| for (const inputMessage of inputMessages) { | ||||||
| const storedParts = storedTerminalParts.get(inputMessage.id); | ||||||
| if (!storedParts) continue; | ||||||
| inputMessage.parts = ((inputMessage.parts ?? []) as any[]).map((part) => { | ||||||
| if ( | ||||||
| typeof part?.type === "string" && | ||||||
| part.type.startsWith("tool-") && | ||||||
| part.toolCallId && | ||||||
| !terminalToolStates.has(part.state) | ||||||
| ) { | ||||||
| const storedPart = storedParts.get(part.toolCallId); | ||||||
| if (storedPart) { | ||||||
| return JSON.parse(JSON.stringify(storedPart)); | ||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P3: This new Prompt for AI agents
Suggested change
|
||||||
| } | ||||||
| } | ||||||
| return part; | ||||||
| }); | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| if (idsToReplace.size > 0) { | ||||||
| for (let index = messages.length - 1; index >= 0; index--) { | ||||||
| if (idsToReplace.has(messages[index].id)) { | ||||||
|
|
@@ -6548,6 +6674,8 @@ export class Agent { | |||||
| const finalOutput = await resolveToolEndOutput(validatedResult); | ||||||
| spanOutcome = { status: "completed", output: finalOutput }; | ||||||
|
|
||||||
| await this.persistApprovedToolResult({ tool, toolCallId, output: finalOutput, oc }); | ||||||
|
|
||||||
| return finalOutput; | ||||||
| }; | ||||||
|
|
||||||
|
|
||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,139 @@ | ||
| /** | ||
| * Regression tests for https://github.com/VoltAgent/voltagent/issues/1535 | ||
| * | ||
| * After a `needsApproval` tool call is approved and runs, memory must store | ||
| * the tool part as `state: "output-available"` WITH `approval.approved` | ||
| * preserved. Previously the buffer was seeded only from stored memory (which | ||
| * still held the pre-approval part, `approval: { id }`), the approval | ||
| * response in the input never reached the buffer, and the flush overwrote | ||
| * the input message - so the stored part lost `approved: true` and every | ||
| * later `generateText` on the conversation threw AI_TypeValidationError. | ||
| */ | ||
| import { MockLanguageModelV3 } from "ai/test"; | ||
| import { describe, expect, it } from "vitest"; | ||
| import { z } from "zod"; | ||
| import { Memory } from "../memory"; | ||
| import { InMemoryStorageAdapter } from "../memory/adapters/storage/in-memory"; | ||
| import { createTool } from "../tool"; | ||
| import { Agent } from "./agent"; | ||
|
|
||
| const usage = { | ||
| inputTokens: 10, | ||
| outputTokens: 5, | ||
| totalTokens: 15, | ||
| inputTokenDetails: { | ||
| noCacheTokens: 10, | ||
| cacheReadTokens: 0, | ||
| cacheWriteTokens: 0, | ||
| }, | ||
| outputTokenDetails: { | ||
| textTokens: 5, | ||
| reasoningTokens: 0, | ||
| }, | ||
| }; | ||
|
|
||
| const makeAgent = (storage: InMemoryStorageAdapter) => { | ||
| const model = new MockLanguageModelV3({ | ||
| modelId: "test-model", | ||
| doGenerate: async ({ prompt }) => { | ||
| await new Promise((r) => setTimeout(r, 1)); | ||
| return prompt.some((m: any) => m.role === "tool") | ||
| ? { | ||
| content: [{ type: "text" as const, text: "Refund done." }], | ||
| finishReason: { unified: "stop" as const, raw: "stop" }, | ||
| usage, | ||
| warnings: [], | ||
| } | ||
| : { | ||
| content: [ | ||
| { | ||
| type: "tool-call" as const, | ||
| toolCallId: "call_1", | ||
| toolName: "refund", | ||
| input: '{"amount":50}', | ||
| }, | ||
| ], | ||
| finishReason: { unified: "tool-calls" as const, raw: "tool_use" }, | ||
| usage, | ||
| warnings: [], | ||
| }; | ||
| }, | ||
| }); | ||
|
|
||
| return new Agent({ | ||
| name: "a", | ||
| instructions: "x", | ||
| model: model as any, | ||
| memory: new Memory({ storage }), | ||
| tools: [ | ||
| createTool({ | ||
| name: "refund", | ||
| description: "refund", | ||
| parameters: z.object({ amount: z.number() }), | ||
| needsApproval: true, | ||
| execute: async () => ({ refunded: true }), | ||
| }), | ||
| ], | ||
| }); | ||
| }; | ||
|
|
||
| const memory = { userId: "u", conversationId: "c" }; | ||
|
|
||
| const storedToolParts = async (storage: InMemoryStorageAdapter) => { | ||
| const stored = await storage.getMessages("u", "c"); | ||
| return stored.flatMap((m: any) => m.parts ?? []).filter((p: any) => p.type === "tool-refund"); | ||
| }; | ||
|
|
||
| describe("needsApproval persistence (issue #1535)", () => { | ||
| it("keeps approval.approved when an approved tool's result is persisted, and later turns work", async () => { | ||
| const storage = new InMemoryStorageAdapter(); | ||
| const agent = makeAgent(storage); | ||
|
|
||
| // Turn 1: model calls the tool, agent asks for approval. | ||
| await agent.generateText("refund 50", { memory }); | ||
|
|
||
| const stored1 = await storage.getMessages("u", "c"); | ||
| const msg = structuredClone(stored1.at(-1)) as any; | ||
| const part = msg.parts.find((p: any) => p.type === "tool-refund"); | ||
| expect(part.state).toBe("approval-requested"); | ||
|
|
||
| // Client approves (shape produced by the AI SDK's addToolApprovalResponse). | ||
| part.state = "approval-responded"; | ||
| part.approval = { id: part.approval.id, approved: true }; | ||
|
|
||
| // Turn 2: tool runs. | ||
| await agent.generateText([msg], { memory }); | ||
|
|
||
| const parts2 = await storedToolParts(storage); | ||
| expect(parts2).toHaveLength(1); | ||
| expect(parts2[0].state).toBe("output-available"); | ||
| expect(parts2[0].output).toBeDefined(); | ||
| expect(parts2[0].approval).toMatchObject({ approved: true }); | ||
|
|
||
| // Turn 3: a plain follow-up must not fail AI SDK UI message validation. | ||
| const result = await agent.generateText("did it go through?", { memory }); | ||
| expect(result.text).toBeDefined(); | ||
| }, 30000); | ||
|
|
||
| it("keeps approval.approved === false when the client denies the tool", async () => { | ||
| const storage = new InMemoryStorageAdapter(); | ||
| const agent = makeAgent(storage); | ||
|
|
||
| await agent.generateText("refund 50", { memory }); | ||
|
|
||
| const stored1 = await storage.getMessages("u", "c"); | ||
| const msg = structuredClone(stored1.at(-1)) as any; | ||
| const part = msg.parts.find((p: any) => p.type === "tool-refund"); | ||
|
|
||
| // Client denies. | ||
| part.state = "approval-responded"; | ||
| part.approval = { id: part.approval.id, approved: false }; | ||
|
|
||
| await agent.generateText([msg], { memory }); | ||
|
|
||
| const parts2 = await storedToolParts(storage); | ||
| expect(parts2).toHaveLength(1); | ||
| // The client's denial decision is preserved, never rewritten to an approval. | ||
| expect(parts2[0].approval?.approved).toBe(false); | ||
| }, 30000); | ||
| }); |
Uh oh!
There was an error while loading. Please reload this page.