fix(ralph-loop): detect oracle VERIFIED tool results in session messages
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
@@ -89,7 +89,7 @@ describe("detectCompletionInSessionMessages", () => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
describe("#given promise appears in tool_result part (not text part)", () => {
|
describe("#given promise appears in tool_result part (not text part)", () => {
|
||||||
test("#when Oracle returns VERIFIED via task() tool_result #then should NOT detect completion", async () => {
|
test("#when Oracle returns VERIFIED via task() tool_result #then should detect completion", async () => {
|
||||||
const messages = [
|
const messages = [
|
||||||
{
|
{
|
||||||
info: { role: "assistant" },
|
info: { role: "assistant" },
|
||||||
@@ -116,6 +116,29 @@ describe("detectCompletionInSessionMessages", () => {
|
|||||||
sinceMessageIndex: 0,
|
sinceMessageIndex: 0,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
expect(detected).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("#when non-Oracle tool_result returns VERIFIED #then should NOT detect completion", async () => {
|
||||||
|
const messages = [
|
||||||
|
{
|
||||||
|
info: { role: "assistant" },
|
||||||
|
parts: [
|
||||||
|
{ type: "tool_result", text: "Agent: explore\n\n<promise>VERIFIED</promise>" },
|
||||||
|
{ type: "text", text: "Explore finished checking." },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
const ctx = createPluginInput(messages)
|
||||||
|
|
||||||
|
const detected = await detectCompletionInSessionMessages(ctx, {
|
||||||
|
sessionID: "session-123",
|
||||||
|
promise: "VERIFIED",
|
||||||
|
apiTimeoutMs: 1000,
|
||||||
|
directory: "/tmp",
|
||||||
|
sinceMessageIndex: 0,
|
||||||
|
})
|
||||||
|
|
||||||
expect(detected).toBe(false)
|
expect(detected).toBe(false)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@@ -34,6 +34,22 @@ function buildPromisePattern(promise: string): RegExp {
|
|||||||
return new RegExp(`<promise>\\s*${escapeRegex(promise)}\\s*</promise>`, "is")
|
return new RegExp(`<promise>\\s*${escapeRegex(promise)}\\s*</promise>`, "is")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function shouldInspectSessionMessagePart(
|
||||||
|
partType: string,
|
||||||
|
promise: string,
|
||||||
|
partText: string,
|
||||||
|
): boolean {
|
||||||
|
if (partType === "text") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
if (partType !== "tool_result") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return promise === ULTRAWORK_VERIFICATION_PROMISE && ORACLE_AGENT_PATTERN.test(partText)
|
||||||
|
}
|
||||||
|
|
||||||
function shouldInspectTranscriptEntry(
|
function shouldInspectTranscriptEntry(
|
||||||
entry: TranscriptEntry,
|
entry: TranscriptEntry,
|
||||||
promise: string,
|
promise: string,
|
||||||
@@ -127,14 +143,13 @@ export async function detectCompletionInSessionMessages(
|
|||||||
const assistant = assistantMessages[index]
|
const assistant = assistantMessages[index]
|
||||||
if (!assistant.parts) continue
|
if (!assistant.parts) continue
|
||||||
|
|
||||||
let responseText = ""
|
|
||||||
for (const part of assistant.parts) {
|
for (const part of assistant.parts) {
|
||||||
if (part.type !== "text") continue
|
const partText = part.text ?? ""
|
||||||
responseText += `${responseText ? "\n" : ""}${part.text ?? ""}`
|
if (!partText) continue
|
||||||
}
|
if (!shouldInspectSessionMessagePart(part.type, options.promise, partText)) continue
|
||||||
|
if (pattern.test(partText)) {
|
||||||
if (pattern.test(responseText)) {
|
return true
|
||||||
return true
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user