fix(ralph-loop): detect promise tags in tool_result parts for ulw verification

Oracle's <promise>VERIFIED</promise> arrives as a tool_result part from the
task() tool call, not as a text part. Both detectCompletionInSessionMessages
and collectAssistantText only scanned type=text parts, missing the
verification signal entirely. This caused ulw loops to fail verification
even when Oracle successfully emitted VERIFIED.

Include tool_result parts in promise detection alongside text parts.
Exclude tool_use parts to avoid false positives from instructional text.
This commit is contained in:
YeonGyu-Kim
2026-03-18 19:00:58 +09:00
parent 22fd976eb9
commit f6c24e42af
3 changed files with 78 additions and 2 deletions
@@ -108,4 +108,80 @@ describe("detectCompletionInSessionMessages", () => {
expect(detected).toBe(true)
})
})
describe("#given promise appears in tool_result part (not text part)", () => {
test("#when Oracle returns VERIFIED via task() tool_result #then should detect completion", async () => {
const messages: SessionMessage[] = [
{
info: { role: "assistant" },
parts: [
{ type: "text", text: "Consulting Oracle for verification." },
{ type: "tool_use", text: '{"subagent_type":"oracle"}' },
],
},
{
info: { role: "assistant" },
parts: [
{ type: "tool_result", text: 'Task completed.\n\nAgent: oracle\n\n<promise>VERIFIED</promise>\n\n<task_metadata>\nsession_id: ses_abc123\n</task_metadata>' },
{ type: "text", text: "Oracle verified the task." },
],
},
]
const ctx = createPluginInput(messages)
const detected = await detectCompletionInSessionMessages(ctx, {
sessionID: "session-123",
promise: "VERIFIED",
apiTimeoutMs: 1000,
directory: "/tmp",
sinceMessageIndex: 0,
})
expect(detected).toBe(true)
})
test("#when DONE appears only in tool_result part #then should detect completion", async () => {
const messages: SessionMessage[] = [
{
info: { role: "assistant" },
parts: [
{ type: "tool_result", text: 'Background task output <promise>DONE</promise>' },
{ type: "text", text: "Task completed successfully." },
],
},
]
const ctx = createPluginInput(messages)
const detected = await detectCompletionInSessionMessages(ctx, {
sessionID: "session-123",
promise: "DONE",
apiTimeoutMs: 1000,
directory: "/tmp",
})
expect(detected).toBe(true)
})
test("#when promise appears in tool_use part (not tool_result) #then should NOT detect completion", async () => {
const messages: SessionMessage[] = [
{
info: { role: "assistant" },
parts: [
{ type: "tool_use", text: 'prompt containing <promise>VERIFIED</promise> as instruction' },
{ type: "text", text: "Calling Oracle." },
],
},
]
const ctx = createPluginInput(messages)
const detected = await detectCompletionInSessionMessages(ctx, {
sessionID: "session-123",
promise: "VERIFIED",
apiTimeoutMs: 1000,
directory: "/tmp",
})
expect(detected).toBe(false)
})
})
})
@@ -93,7 +93,7 @@ export async function detectCompletionInSessionMessages(
let responseText = ""
for (const part of assistant.parts) {
if (part.type !== "text") continue
if (part.type !== "text" && part.type !== "tool_result") continue
responseText += `${responseText ? "\n" : ""}${part.text ?? ""}`
}
@@ -25,7 +25,7 @@ function collectAssistantText(message: OpenCodeSessionMessage): string {
let text = ""
for (const part of message.parts) {
if (part.type !== "text") {
if (part.type !== "text" && part.type !== "tool_result") {
continue
}
text += `${text ? "\n" : ""}${part.text ?? ""}`