fix(ralph-loop): harden oracle verification flow
Capture oracle verification sessions more reliably and accept parent-session VERIFIED evidence so ULW loops do not retry after successful review. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
@@ -25,6 +25,8 @@ You already emitted <promise>{{INITIAL_PROMISE}}</promise>. This does NOT finish
|
||||
REQUIRED NOW:
|
||||
- Call Oracle using task(subagent_type="oracle", load_skills=[], run_in_background=false, ...)
|
||||
- Ask Oracle to verify whether the original task is actually complete
|
||||
- Include the original task in the Oracle request
|
||||
- Explicitly tell Oracle to review skeptically and critically, and to look for reasons the task may still be incomplete or wrong
|
||||
- The system will inspect the Oracle session directly for the verification result
|
||||
- If Oracle does not verify, continue fixing the task and do not consider it complete
|
||||
|
||||
@@ -40,6 +42,7 @@ REQUIRED NOW:
|
||||
- Oracle does not lie. Treat the verification result as ground truth
|
||||
- Do not claim completion early or argue with the failed verification
|
||||
- After fixing the remaining issues, request Oracle review again using task(subagent_type="oracle", load_skills=[], run_in_background=false, ...)
|
||||
- Include the original task in the Oracle request and tell Oracle to review skeptically and critically
|
||||
- Only when the work is ready for review again, output: <promise>{{PROMISE}}</promise>
|
||||
|
||||
Original task:
|
||||
|
||||
@@ -87,7 +87,7 @@ export function createRalphLoopEventHandler(
|
||||
return
|
||||
}
|
||||
|
||||
const completionSessionID = verificationSessionID ?? (state.verification_pending ? undefined : sessionID)
|
||||
const completionSessionID = verificationSessionID ?? sessionID
|
||||
const transcriptPath = completionSessionID ? options.getTranscriptPath(completionSessionID) : undefined
|
||||
const completionViaTranscript = completionSessionID
|
||||
? detectCompletionInTranscript(
|
||||
@@ -107,7 +107,13 @@ export function createRalphLoopEventHandler(
|
||||
sinceMessageIndex: undefined,
|
||||
})
|
||||
: state.verification_pending
|
||||
? false
|
||||
? await detectCompletionInSessionMessages(ctx, {
|
||||
sessionID,
|
||||
promise: state.completion_promise,
|
||||
apiTimeoutMs: options.apiTimeoutMs,
|
||||
directory: options.directory,
|
||||
sinceMessageIndex: state.message_count_at_start,
|
||||
})
|
||||
: await detectCompletionInSessionMessages(ctx, {
|
||||
sessionID,
|
||||
promise: state.completion_promise,
|
||||
|
||||
@@ -366,7 +366,7 @@ describe("ulw-loop verification", () => {
|
||||
expect(toastCalls.some((toast) => toast.title === "ULTRAWORK LOOP COMPLETE!")).toBe(false)
|
||||
})
|
||||
|
||||
test("#given parent session emits VERIFIED #when oracle session is not tracked #then ulw loop continues instead of completing", async () => {
|
||||
test("#given parent session emits VERIFIED #when oracle session is not tracked #then ulw loop completes from parent session evidence", async () => {
|
||||
const hook = createRalphLoopHook(createMockPluginInput(), {
|
||||
getTranscriptPath: (sessionID) => sessionID === "ses-oracle" ? oracleTranscriptPath : parentTranscriptPath,
|
||||
})
|
||||
@@ -379,17 +379,13 @@ describe("ulw-loop verification", () => {
|
||||
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
|
||||
writeFileSync(
|
||||
parentTranscriptPath,
|
||||
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done <promise>DONE</promise>" } })}\n${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: `bad parent leak <promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>` } })}\n`,
|
||||
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done <promise>DONE</promise>" } })}\n${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: `verified <promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>` } })}\n`,
|
||||
)
|
||||
|
||||
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
|
||||
|
||||
expect(hook.getState()).not.toBeNull()
|
||||
expect(hook.getState()?.iteration).toBe(2)
|
||||
expect(hook.getState()?.completion_promise).toBe("DONE")
|
||||
expect(hook.getState()?.verification_pending).toBeUndefined()
|
||||
expect(promptCalls).toHaveLength(2)
|
||||
expect(promptCalls[1]?.text).toContain("Verification failed")
|
||||
expect(hook.getState()).toBeNull()
|
||||
expect(toastCalls.some((toast) => toast.title === "ULTRAWORK LOOP COMPLETE!")).toBe(true)
|
||||
})
|
||||
|
||||
test("#given oracle verification fails #when loop restarts #then old oracle session is aborted", async () => {
|
||||
|
||||
Reference in New Issue
Block a user