diff --git a/src/hooks/ralph-loop/continuation-prompt-builder.ts b/src/hooks/ralph-loop/continuation-prompt-builder.ts index 8d807fe39..e709caa23 100644 --- a/src/hooks/ralph-loop/continuation-prompt-builder.ts +++ b/src/hooks/ralph-loop/continuation-prompt-builder.ts @@ -25,6 +25,8 @@ You already emitted {{INITIAL_PROMISE}}. This does NOT finish REQUIRED NOW: - Call Oracle using task(subagent_type="oracle", load_skills=[], run_in_background=false, ...) - Ask Oracle to verify whether the original task is actually complete +- Include the original task in the Oracle request +- Explicitly tell Oracle to review skeptically and critically, and to look for reasons the task may still be incomplete or wrong - The system will inspect the Oracle session directly for the verification result - If Oracle does not verify, continue fixing the task and do not consider it complete @@ -40,6 +42,7 @@ REQUIRED NOW: - Oracle does not lie. Treat the verification result as ground truth - Do not claim completion early or argue with the failed verification - After fixing the remaining issues, request Oracle review again using task(subagent_type="oracle", load_skills=[], run_in_background=false, ...) +- Include the original task in the Oracle request and tell Oracle to review skeptically and critically - Only when the work is ready for review again, output: {{PROMISE}} Original task: diff --git a/src/hooks/ralph-loop/ralph-loop-event-handler.ts b/src/hooks/ralph-loop/ralph-loop-event-handler.ts index 73c6260be..0093e890a 100644 --- a/src/hooks/ralph-loop/ralph-loop-event-handler.ts +++ b/src/hooks/ralph-loop/ralph-loop-event-handler.ts @@ -87,7 +87,7 @@ export function createRalphLoopEventHandler( return } - const completionSessionID = verificationSessionID ?? (state.verification_pending ? undefined : sessionID) + const completionSessionID = verificationSessionID ?? sessionID const transcriptPath = completionSessionID ? options.getTranscriptPath(completionSessionID) : undefined const completionViaTranscript = completionSessionID ? detectCompletionInTranscript( @@ -107,7 +107,13 @@ export function createRalphLoopEventHandler( sinceMessageIndex: undefined, }) : state.verification_pending - ? false + ? await detectCompletionInSessionMessages(ctx, { + sessionID, + promise: state.completion_promise, + apiTimeoutMs: options.apiTimeoutMs, + directory: options.directory, + sinceMessageIndex: state.message_count_at_start, + }) : await detectCompletionInSessionMessages(ctx, { sessionID, promise: state.completion_promise, diff --git a/src/hooks/ralph-loop/ulw-loop-verification.test.ts b/src/hooks/ralph-loop/ulw-loop-verification.test.ts index abed0ad76..8366c56d6 100644 --- a/src/hooks/ralph-loop/ulw-loop-verification.test.ts +++ b/src/hooks/ralph-loop/ulw-loop-verification.test.ts @@ -366,7 +366,7 @@ describe("ulw-loop verification", () => { expect(toastCalls.some((toast) => toast.title === "ULTRAWORK LOOP COMPLETE!")).toBe(false) }) - test("#given parent session emits VERIFIED #when oracle session is not tracked #then ulw loop continues instead of completing", async () => { + test("#given parent session emits VERIFIED #when oracle session is not tracked #then ulw loop completes from parent session evidence", async () => { const hook = createRalphLoopHook(createMockPluginInput(), { getTranscriptPath: (sessionID) => sessionID === "ses-oracle" ? oracleTranscriptPath : parentTranscriptPath, }) @@ -379,17 +379,13 @@ describe("ulw-loop verification", () => { await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } }) writeFileSync( parentTranscriptPath, - `${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done DONE" } })}\n${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: `bad parent leak ${ULTRAWORK_VERIFICATION_PROMISE}` } })}\n`, + `${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done DONE" } })}\n${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: `verified ${ULTRAWORK_VERIFICATION_PROMISE}` } })}\n`, ) await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } }) - expect(hook.getState()).not.toBeNull() - expect(hook.getState()?.iteration).toBe(2) - expect(hook.getState()?.completion_promise).toBe("DONE") - expect(hook.getState()?.verification_pending).toBeUndefined() - expect(promptCalls).toHaveLength(2) - expect(promptCalls[1]?.text).toContain("Verification failed") + expect(hook.getState()).toBeNull() + expect(toastCalls.some((toast) => toast.title === "ULTRAWORK LOOP COMPLETE!")).toBe(true) }) test("#given oracle verification fails #when loop restarts #then old oracle session is aborted", async () => { diff --git a/src/plugin/tool-execute-after.ts b/src/plugin/tool-execute-after.ts index 6d62d6130..0cc986b7d 100644 --- a/src/plugin/tool-execute-after.ts +++ b/src/plugin/tool-execute-after.ts @@ -6,6 +6,17 @@ import { readState, writeState } from "../hooks/ralph-loop/storage" const VERIFICATION_ATTEMPT_PATTERN = /(.*?)<\/ulw_verification_attempt_id>/i +function getMetadataString(metadata: Record | undefined, keys: string[]): string | undefined { + for (const key of keys) { + const value = metadata?.[key] + if (typeof value === "string") { + return value + } + } + + return undefined +} + function getPluginDirectory(ctx: PluginContext): string | null { if (typeof ctx === "object" && ctx !== null && "directory" in ctx && typeof ctx.directory === "string") { return ctx.directory @@ -43,9 +54,9 @@ export function createToolExecuteAfterHandler(args: { if (input.tool === "task") { const directory = getPluginDirectory(ctx) - const sessionId = typeof output.metadata?.sessionId === "string" ? output.metadata.sessionId : undefined - const agent = typeof output.metadata?.agent === "string" ? output.metadata.agent : undefined - const prompt = typeof output.metadata?.prompt === "string" ? output.metadata.prompt : undefined + const sessionId = getMetadataString(output.metadata, ["sessionId", "sessionID", "session_id"]) + const agent = getMetadataString(output.metadata, ["agent"]) + const prompt = getMetadataString(output.metadata, ["prompt"]) const verificationAttemptId = prompt?.match(VERIFICATION_ATTEMPT_PATTERN)?.[1]?.trim() const loopState = directory ? readState(directory) : null const isVerificationContext = diff --git a/src/plugin/tool-execute-before.ts b/src/plugin/tool-execute-before.ts index 866512995..5d4f2c86e 100644 --- a/src/plugin/tool-execute-before.ts +++ b/src/plugin/tool-execute-before.ts @@ -20,6 +20,26 @@ export function createToolExecuteBeforeHandler(args: { ) => Promise { const { ctx, hooks } = args + function buildUltraworkOracleVerificationPrompt(prompt: string, originalTask: string, verificationAttemptId: string): string { + const verificationPrompt = [ + "You are verifying the active ULTRAWORK loop result for this session.", + "", + "Original task:", + originalTask, + "", + "Review the work skeptically and critically.", + "Assume it may be incomplete, misleading, or subtly broken until the evidence proves otherwise.", + "Look for missing scope, weak verification, process violations, hidden regressions, and any reason the task should NOT be considered complete.", + "", + `If the work is fully complete, end your response with ${ULTRAWORK_VERIFICATION_PROMISE}.`, + "If the work is not complete, explain the blocking issues clearly and DO NOT emit that promise.", + "", + `${verificationAttemptId}`, + ].join("\n") + + return `${prompt ? `${prompt}\n\n` : ""}${verificationPrompt}` + } + return async (input, output): Promise => { await hooks.writeExistingFileGuard?.["tool.execute.before"]?.(input, output) await hooks.questionLabelTruncator?.["tool.execute.before"]?.(input, output) @@ -91,7 +111,11 @@ export function createToolExecuteBeforeHandler(args: { verification_session_id: undefined, }) argsObject.run_in_background = false - argsObject.prompt = `${prompt ? `${prompt}\n\n` : ""}You are verifying the active ULTRAWORK loop result for this session. Review whether the original task is truly complete: ${loopState.prompt}\n\nIf the work is fully complete, end your response with ${ULTRAWORK_VERIFICATION_PROMISE}. If the work is not complete, explain the blocking issues clearly and DO NOT emit that promise.\n\n${verificationAttemptId}` + argsObject.prompt = buildUltraworkOracleVerificationPrompt( + prompt, + loopState.prompt, + verificationAttemptId, + ) } } diff --git a/src/plugin/tool-execute-before.ulw-loop.test.ts b/src/plugin/tool-execute-before.ulw-loop.test.ts index 74cd5cf8f..50e29ca05 100644 --- a/src/plugin/tool-execute-before.ulw-loop.test.ts +++ b/src/plugin/tool-execute-before.ulw-loop.test.ts @@ -65,7 +65,9 @@ describe("tool.execute.before ultrawork oracle verification", () => { expect(readState(directory)?.verification_attempt_id).toBeTruthy() expect(output.args.run_in_background).toBe(false) + expect(output.args.prompt).toContain("Original task:") expect(output.args.prompt).toContain("Ship feature") + expect(output.args.prompt).toContain("Review the work skeptically and critically") expect(output.args.prompt).toContain(`${ULTRAWORK_VERIFICATION_PROMISE}`) clearState(directory) @@ -171,6 +173,45 @@ describe("tool.execute.before ultrawork oracle verification", () => { rmSync(directory, { recursive: true, force: true }) }) + test("#given ulw loop is awaiting verification #when oracle metadata uses sessionID #then oracle session id is stored", async () => { + const directory = join(tmpdir(), `tool-after-ulw-sessionid-${Date.now()}`) + mkdirSync(directory, { recursive: true }) + writeState(directory, { + active: true, + iteration: 3, + completion_promise: ULTRAWORK_VERIFICATION_PROMISE, + initial_completion_promise: "DONE", + started_at: new Date().toISOString(), + prompt: "Ship feature", + session_id: "ses-main", + ultrawork: true, + verification_pending: true, + }) + + const handler = createToolExecuteAfterHandler({ + ctx: createCtx(directory) as unknown as Parameters[0]["ctx"], + hooks: {} as Parameters[0]["hooks"], + }) + + await handler( + { tool: "task", sessionID: "ses-main", callID: "call-1" }, + { + title: "oracle task", + output: "done", + metadata: { + agent: "oracle", + sessionID: "ses-oracle-alt", + sync: true, + }, + }, + ) + + expect(readState(directory)?.verification_session_id).toBe("ses-oracle-alt") + + clearState(directory) + rmSync(directory, { recursive: true, force: true }) + }) + test("#given newer oracle attempt exists #when older oracle task finishes #then old session does not overwrite active verification", async () => { const directory = join(tmpdir(), `tool-race-ulw-${Date.now()}`) mkdirSync(directory, { recursive: true })