From f31537f14c1a12b5a4cc9704794ca17b1357be86 Mon Sep 17 00:00:00 2001 From: YeonGyu-Kim Date: Thu, 12 Mar 2026 17:00:25 +0900 Subject: [PATCH 1/2] fix(ralph-loop): continue ultrawork until oracle verifies Keep /ulw-loop iterating after the main session emits DONE so completion still depends on an actual Oracle VERIFIED result. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .../pending-verification-handler.ts | 52 +++++++++++++++++++ .../ralph-loop/ralph-loop-event-handler.ts | 23 +++----- .../ralph-loop/ulw-loop-verification.test.ts | 33 ++++++++---- 3 files changed, 83 insertions(+), 25 deletions(-) create mode 100644 src/hooks/ralph-loop/pending-verification-handler.ts diff --git a/src/hooks/ralph-loop/pending-verification-handler.ts b/src/hooks/ralph-loop/pending-verification-handler.ts new file mode 100644 index 000000000..065e07d4c --- /dev/null +++ b/src/hooks/ralph-loop/pending-verification-handler.ts @@ -0,0 +1,52 @@ +import type { PluginInput } from "@opencode-ai/plugin" +import { log } from "../../shared/logger" +import { HOOK_NAME } from "./constants" +import type { RalphLoopState } from "./types" +import { handleFailedVerification } from "./verification-failure-handler" + +type LoopStateController = { + restartAfterFailedVerification: (sessionID: string, messageCountAtStart?: number) => RalphLoopState | null +} + +export async function handlePendingVerification( + ctx: PluginInput, + input: { + sessionID: string + state: RalphLoopState + verificationSessionID?: string + matchesParentSession: boolean + matchesVerificationSession: boolean + loopState: LoopStateController + directory: string + apiTimeoutMs: number + }, +): Promise { + const { + sessionID, + state, + verificationSessionID, + matchesParentSession, + matchesVerificationSession, + loopState, + directory, + apiTimeoutMs, + } = input + + if (matchesParentSession || (verificationSessionID && matchesVerificationSession)) { + const restarted = await handleFailedVerification(ctx, { + state, + loopState, + directory, + apiTimeoutMs, + }) + if (restarted) { + return + } + } + + log(`[${HOOK_NAME}] Waiting for oracle verification`, { + sessionID, + verificationSessionID, + iteration: state.iteration, + }) +} diff --git a/src/hooks/ralph-loop/ralph-loop-event-handler.ts b/src/hooks/ralph-loop/ralph-loop-event-handler.ts index bd41ce5b6..e4a53b792 100644 --- a/src/hooks/ralph-loop/ralph-loop-event-handler.ts +++ b/src/hooks/ralph-loop/ralph-loop-event-handler.ts @@ -8,8 +8,8 @@ import { detectCompletionInTranscript, } from "./completion-promise-detector" import { continueIteration } from "./iteration-continuation" +import { handlePendingVerification } from "./pending-verification-handler" import { handleDeletedLoopSession, handleErroredLoopSession } from "./session-event-handler" -import { handleFailedVerification } from "./verification-failure-handler" type SessionRecovery = { isRecovering: (sessionID: string) => boolean @@ -136,22 +136,15 @@ export function createRalphLoopEventHandler( } if (state.verification_pending) { - if (verificationSessionID && matchesVerificationSession) { - const restarted = await handleFailedVerification(ctx, { - state, - loopState: options.loopState, - directory: options.directory, - apiTimeoutMs: options.apiTimeoutMs, - }) - if (restarted) { - return - } - } - - log(`[${HOOK_NAME}] Waiting for oracle verification`, { + await handlePendingVerification(ctx, { sessionID, + state, verificationSessionID, - iteration: state.iteration, + matchesParentSession, + matchesVerificationSession, + loopState: options.loopState, + directory: options.directory, + apiTimeoutMs: options.apiTimeoutMs, }) return } diff --git a/src/hooks/ralph-loop/ulw-loop-verification.test.ts b/src/hooks/ralph-loop/ulw-loop-verification.test.ts index 2018964b4..a1b08e44e 100644 --- a/src/hooks/ralph-loop/ulw-loop-verification.test.ts +++ b/src/hooks/ralph-loop/ulw-loop-verification.test.ts @@ -129,7 +129,7 @@ describe("ulw-loop verification", () => { expect(toastCalls.some((toast) => toast.title === "ULTRAWORK LOOP COMPLETE!")).toBe(true) }) - test("#given ulw loop is awaiting verification without oracle session #when idle fires again #then loop waits instead of continuing", async () => { + test("#given ulw loop is awaiting verification without oracle session #when parent idles again #then loop continues until oracle verifies", async () => { const hook = createRalphLoopHook(createMockPluginInput(), { getTranscriptPath: (sessionID) => sessionID === "ses-oracle" ? oracleTranscriptPath : parentTranscriptPath, }) @@ -144,12 +144,16 @@ describe("ulw-loop verification", () => { await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } }) - expect(hook.getState()?.iteration).toBe(stateAfterDone?.iteration) - expect(promptCalls).toHaveLength(1) - expect(hook.getState()?.verification_pending).toBe(true) + expect(stateAfterDone?.verification_pending).toBe(true) + expect(hook.getState()?.iteration).toBe(2) + expect(hook.getState()?.completion_promise).toBe("DONE") + expect(hook.getState()?.verification_pending).toBeUndefined() + expect(promptCalls).toHaveLength(2) + expect(promptCalls[1]?.sessionID).toBe("session-123") + expect(promptCalls[1]?.text).toContain("Verification failed") }) - test("#given ulw loop is awaiting oracle verification #when oracle has not verified yet #then loop waits instead of continuing", async () => { + test("#given ulw loop is awaiting oracle verification #when parent idles before VERIFIED arrives #then loop continues instead of waiting", async () => { const hook = createRalphLoopHook(createMockPluginInput(), { getTranscriptPath: (sessionID) => sessionID === "ses-oracle" ? oracleTranscriptPath : parentTranscriptPath, }) @@ -172,9 +176,14 @@ describe("ulw-loop verification", () => { await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } }) - expect(hook.getState()?.iteration).toBe(stateBeforeWait?.iteration) - expect(promptCalls).toHaveLength(1) - expect(hook.getState()?.verification_session_id).toBe("ses-oracle") + expect(stateBeforeWait?.verification_session_id).toBe("ses-oracle") + expect(hook.getState()?.iteration).toBe(2) + expect(hook.getState()?.completion_promise).toBe("DONE") + expect(hook.getState()?.verification_pending).toBeUndefined() + expect(hook.getState()?.verification_session_id).toBeUndefined() + expect(promptCalls).toHaveLength(2) + expect(promptCalls[1]?.sessionID).toBe("session-123") + expect(promptCalls[1]?.text).toContain("Verification failed") }) test("#given oracle verification fails #when oracle session idles #then main session receives retry instructions", async () => { @@ -273,7 +282,7 @@ describe("ulw-loop verification", () => { expect(hook.getState()?.completion_promise).toBe("DONE") }) - test("#given parent session emits VERIFIED #when oracle session is not tracked #then ulw loop does not complete", async () => { + test("#given parent session emits VERIFIED #when oracle session is not tracked #then ulw loop continues instead of completing", async () => { const hook = createRalphLoopHook(createMockPluginInput(), { getTranscriptPath: (sessionID) => sessionID === "ses-oracle" ? oracleTranscriptPath : parentTranscriptPath, }) @@ -292,6 +301,10 @@ describe("ulw-loop verification", () => { await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } }) expect(hook.getState()).not.toBeNull() - expect(hook.getState()?.verification_pending).toBe(true) + expect(hook.getState()?.iteration).toBe(2) + expect(hook.getState()?.completion_promise).toBe("DONE") + expect(hook.getState()?.verification_pending).toBeUndefined() + expect(promptCalls).toHaveLength(2) + expect(promptCalls[1]?.text).toContain("Verification failed") }) }) From 1812c9f054a488220284adf57e9ad1a43aad837a Mon Sep 17 00:00:00 2001 From: YeonGyu-Kim Date: Thu, 12 Mar 2026 17:05:02 +0900 Subject: [PATCH 2/2] test(ralph-loop): cover overlapping ultrawork loops Lock down stale-session and overwrite cases so a previous ULW verification flow cannot complete or mutate a newer loop. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .../ralph-loop/ulw-loop-verification.test.ts | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/src/hooks/ralph-loop/ulw-loop-verification.test.ts b/src/hooks/ralph-loop/ulw-loop-verification.test.ts index a1b08e44e..6e411a4d3 100644 --- a/src/hooks/ralph-loop/ulw-loop-verification.test.ts +++ b/src/hooks/ralph-loop/ulw-loop-verification.test.ts @@ -282,6 +282,84 @@ describe("ulw-loop verification", () => { expect(hook.getState()?.completion_promise).toBe("DONE") }) + test("#given ulw loop was awaiting verification #when different session starts a new ulw loop #then prior verification state is overwritten", async () => { + const hook = createRalphLoopHook(createMockPluginInput(), { + getTranscriptPath: (sessionID) => sessionID === "ses-oracle" ? oracleTranscriptPath : parentTranscriptPath, + }) + hook.startLoop("session-123", "Build API", { ultrawork: true }) + writeFileSync( + parentTranscriptPath, + `${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done DONE" } })}\n`, + ) + + await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } }) + hook.startLoop("session-456", "Ship CLI", { ultrawork: true }) + + expect(hook.getState()?.session_id).toBe("session-456") + expect(hook.getState()?.prompt).toBe("Ship CLI") + expect(hook.getState()?.verification_pending).toBeUndefined() + expect(hook.getState()?.completion_promise).toBe("DONE") + }) + + test("#given verification state was overwritten by different ulw loop #when stale oracle session idles #then new loop remains active", async () => { + const hook = createRalphLoopHook(createMockPluginInput(), { + getTranscriptPath: (sessionID) => sessionID === "ses-oracle-old" ? oracleTranscriptPath : parentTranscriptPath, + }) + hook.startLoop("session-123", "Build API", { ultrawork: true }) + writeFileSync( + parentTranscriptPath, + `${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done DONE" } })}\n`, + ) + + await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } }) + writeState(testDir, { + ...hook.getState()!, + verification_session_id: "ses-oracle-old", + }) + hook.startLoop("session-456", "Ship CLI", { ultrawork: true }) + writeFileSync( + oracleTranscriptPath, + `${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: `verified ${ULTRAWORK_VERIFICATION_PROMISE}` } })}\n`, + ) + + await hook.event({ event: { type: "session.idle", properties: { sessionID: "ses-oracle-old" } } }) + + expect(hook.getState()?.session_id).toBe("session-456") + expect(hook.getState()?.prompt).toBe("Ship CLI") + expect(hook.getState()?.iteration).toBe(1) + expect(toastCalls.some((toast) => toast.title === "ULTRAWORK LOOP COMPLETE!")).toBe(false) + }) + + test("#given verification state was overwritten by restarted ulw loop #when stale oracle session idles #then restarted loop remains active", async () => { + const hook = createRalphLoopHook(createMockPluginInput(), { + getTranscriptPath: (sessionID) => sessionID === "ses-oracle-old" ? oracleTranscriptPath : parentTranscriptPath, + }) + hook.startLoop("session-123", "Build API", { ultrawork: true }) + writeFileSync( + parentTranscriptPath, + `${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done DONE" } })}\n`, + ) + + await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } }) + writeState(testDir, { + ...hook.getState()!, + verification_session_id: "ses-oracle-old", + }) + hook.startLoop("session-123", "Restarted task", { ultrawork: true }) + writeFileSync( + oracleTranscriptPath, + `${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: `verified ${ULTRAWORK_VERIFICATION_PROMISE}` } })}\n`, + ) + + await hook.event({ event: { type: "session.idle", properties: { sessionID: "ses-oracle-old" } } }) + + expect(hook.getState()?.session_id).toBe("session-123") + expect(hook.getState()?.prompt).toBe("Restarted task") + expect(hook.getState()?.iteration).toBe(1) + expect(hook.getState()?.verification_pending).toBeUndefined() + expect(toastCalls.some((toast) => toast.title === "ULTRAWORK LOOP COMPLETE!")).toBe(false) + }) + test("#given parent session emits VERIFIED #when oracle session is not tracked #then ulw loop continues instead of completing", async () => { const hook = createRalphLoopHook(createMockPluginInput(), { getTranscriptPath: (sessionID) => sessionID === "ses-oracle" ? oracleTranscriptPath : parentTranscriptPath,