diff --git a/src/hooks/ralph-loop/loop-state-controller.ts b/src/hooks/ralph-loop/loop-state-controller.ts index bd7fffcf0..4b54b7a3c 100644 --- a/src/hooks/ralph-loop/loop-state-controller.ts +++ b/src/hooks/ralph-loop/loop-state-controller.ts @@ -45,6 +45,7 @@ export function createLoopStateController(options: { completion_promise: initialCompletionPromise, initial_completion_promise: initialCompletionPromise, verification_attempt_id: undefined, + verification_attempt_started_at: undefined, verification_session_id: undefined, ultrawork: loopOptions?.ultrawork, verification_pending: undefined, @@ -139,6 +140,7 @@ export function createLoopStateController(options: { state.verification_pending = true state.completion_promise = ULTRAWORK_VERIFICATION_PROMISE state.verification_attempt_id = undefined + state.verification_attempt_started_at = undefined state.verification_session_id = undefined state.initial_completion_promise ??= DEFAULT_COMPLETION_PROMISE @@ -156,6 +158,7 @@ export function createLoopStateController(options: { } state.verification_session_id = verificationSessionID + state.verification_attempt_started_at = undefined if (!writeState(directory, state, stateDir)) { return null @@ -175,6 +178,7 @@ export function createLoopStateController(options: { state.completion_promise = state.initial_completion_promise ?? DEFAULT_COMPLETION_PROMISE state.verification_pending = undefined state.verification_attempt_id = undefined + state.verification_attempt_started_at = undefined state.verification_session_id = undefined if (typeof messageCountAtStart === "number") { state.message_count_at_start = messageCountAtStart @@ -197,6 +201,7 @@ export function createLoopStateController(options: { state.completion_promise = state.initial_completion_promise ?? DEFAULT_COMPLETION_PROMISE state.verification_pending = undefined state.verification_attempt_id = undefined + state.verification_attempt_started_at = undefined state.verification_session_id = undefined if (typeof messageCountAtStart === "number") { state.message_count_at_start = messageCountAtStart diff --git a/src/hooks/ralph-loop/pending-verification-handler.ts b/src/hooks/ralph-loop/pending-verification-handler.ts index 5065f2dd1..c093496dc 100644 --- a/src/hooks/ralph-loop/pending-verification-handler.ts +++ b/src/hooks/ralph-loop/pending-verification-handler.ts @@ -7,6 +7,8 @@ import { handleFailedVerification } from "./verification-failure-handler" import { withTimeout } from "./with-timeout" import type { IterationCommitExpectation } from "./types" +export const STUCK_VERIFICATION_TIMEOUT_MS = 30 * 60 * 1000 + type OpenCodeSessionMessage = { info?: { role?: string } parts?: Array<{ type?: string; text?: string }> @@ -138,12 +140,25 @@ export async function handlePendingVerification( } if (state.verification_attempt_id && !state.verification_session_id) { - log(`[${HOOK_NAME}] Skipped verification failure: oracle dispatch in flight`, { - sessionID, - verificationAttemptId: state.verification_attempt_id, - iteration: state.iteration, - }) - return + const startedAt = state.verification_attempt_started_at + const attemptAgeMs = startedAt !== undefined ? Date.now() - startedAt : undefined + const isStuck = attemptAgeMs !== undefined && attemptAgeMs > STUCK_VERIFICATION_TIMEOUT_MS + + if (isStuck) { + log(`[${HOOK_NAME}] Stuck oracle dispatch detected, proceeding to failure handler`, { + sessionID, + verificationAttemptId: state.verification_attempt_id, + attemptAgeMs, + iteration: state.iteration, + }) + } else { + log(`[${HOOK_NAME}] Skipped verification failure: oracle dispatch in flight`, { + sessionID, + verificationAttemptId: state.verification_attempt_id, + iteration: state.iteration, + }) + return + } } const restarted = await handleFailedVerification(ctx, { diff --git a/src/hooks/ralph-loop/storage.ts b/src/hooks/ralph-loop/storage.ts index f5ca06fe2..346128975 100644 --- a/src/hooks/ralph-loop/storage.ts +++ b/src/hooks/ralph-loop/storage.ts @@ -41,6 +41,7 @@ export function readState(directory: string, customPath?: string): RalphLoopStat } const ultrawork = data.ultrawork === true || data.ultrawork === "true" ? true : undefined + const verificationAttemptStartedAt = Number(data.verification_attempt_started_at) const maxIterations = data.max_iterations === undefined || data.max_iterations === "" ? ultrawork @@ -65,6 +66,12 @@ export function readState(directory: string, customPath?: string): RalphLoopStat verification_attempt_id: data.verification_attempt_id ? stripQuotes(data.verification_attempt_id) : undefined, + verification_attempt_started_at: + data.verification_attempt_started_at === undefined || data.verification_attempt_started_at === "" + ? undefined + : Number.isFinite(verificationAttemptStartedAt) + ? verificationAttemptStartedAt + : undefined, verification_session_id: data.verification_session_id ? stripQuotes(data.verification_session_id) : undefined, @@ -106,9 +113,19 @@ export function writeState( const initialCompletionPromiseLine = state.initial_completion_promise ? `initial_completion_promise: "${state.initial_completion_promise}"\n` : "" + const existingState = readState(directory, customPath) + const verificationAttemptStartedAt = state.verification_session_id || !state.verification_attempt_id + ? undefined + : state.verification_attempt_started_at + ?? (existingState?.verification_attempt_id !== state.verification_attempt_id + ? Date.now() + : existingState.verification_attempt_started_at) const verificationAttemptLine = state.verification_attempt_id ? `verification_attempt_id: "${state.verification_attempt_id}"\n` : "" + const verificationAttemptStartedAtLine = typeof verificationAttemptStartedAt === "number" + ? `verification_attempt_started_at: ${verificationAttemptStartedAt}\n` + : "" const verificationSessionLine = state.verification_session_id ? `verification_session_id: "${state.verification_session_id}"\n` : "" @@ -124,7 +141,7 @@ export function writeState( active: ${state.active} iteration: ${state.iteration} ${maxIterationsLine}completion_promise: "${state.completion_promise}" -${initialCompletionPromiseLine}${verificationAttemptLine}${verificationSessionLine}started_at: "${state.started_at}" +${initialCompletionPromiseLine}${verificationAttemptLine}${verificationAttemptStartedAtLine}${verificationSessionLine}started_at: "${state.started_at}" ${sessionIdLine}${ultraworkLine}${verificationPendingLine}${strategyLine}${messageCountAtStartLine}--- ${state.prompt} ` diff --git a/src/hooks/ralph-loop/stuck-oracle-dispatch-recovery.test.ts b/src/hooks/ralph-loop/stuck-oracle-dispatch-recovery.test.ts new file mode 100644 index 000000000..c9ac2911b --- /dev/null +++ b/src/hooks/ralph-loop/stuck-oracle-dispatch-recovery.test.ts @@ -0,0 +1,138 @@ +import type { PluginInput } from "@opencode-ai/plugin" +import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test" +import { unsafeTestValue } from "../../../test-support/unsafe-test-value" +import { releaseAllPromptAsyncReservationsForTesting } from "../shared/prompt-async-gate" +import { handlePendingVerification, STUCK_VERIFICATION_TIMEOUT_MS } from "./pending-verification-handler" +import type { RalphLoopState } from "./types" + +const NOW_MS = 1_800_000_000_000 + +type PendingVerificationInput = Parameters[1] +type LoopStateController = PendingVerificationInput["loopState"] + +function createState(verificationAttemptStartedAt?: number): RalphLoopState { + const state: RalphLoopState = { + active: true, + iteration: 2, + completion_promise: "", + initial_completion_promise: "DONE", + started_at: "2026-01-01T00:00:00.000Z", + prompt: "Ship release blockers", + session_id: "session-123", + ultrawork: true, + verification_pending: true, + verification_attempt_id: "attempt-123", + } + + if (verificationAttemptStartedAt === undefined) { + return state + } + + return { + ...state, + verification_attempt_started_at: verificationAttemptStartedAt, + } +} + +function createPluginInput(promptCalls: string[]): PluginInput { + return unsafeTestValue({ + client: { + session: { + messages: async () => ({ data: [] }), + promptAsync: async (input: unknown) => { + promptCalls.push(JSON.stringify(input) ?? "") + return {} + }, + abort: async () => ({}), + }, + tui: { + showToast: async () => ({}), + }, + }, + directory: "/tmp/ralph-loop-stuck-oracle-test", + }) +} + +function createLoopStateController(state: RalphLoopState) { + const clearVerificationState = mock(() => state) + const incrementIteration = mock(() => state) + const loopState = { + restartAfterFailedVerification: mock(() => null), + clearVerificationState, + incrementIteration, + clear: mock(() => true), + setVerificationSessionID: mock(() => null), + } satisfies LoopStateController + + return { loopState, clearVerificationState, incrementIteration } +} + +async function runPendingVerification(state: RalphLoopState, loopState: LoopStateController, promptCalls: string[]) { + await handlePendingVerification(createPluginInput(promptCalls), { + sessionID: "session-123", + state, + matchesParentSession: true, + matchesVerificationSession: false, + loopState, + directory: "/tmp/ralph-loop-stuck-oracle-test", + apiTimeoutMs: 100, + }) +} + +describe("ralph-loop stuck oracle dispatch recovery", () => { + const realDateNow = Date.now + + beforeEach(() => { + Date.now = () => NOW_MS + }) + + afterEach(() => { + Date.now = realDateNow + releaseAllPromptAsyncReservationsForTesting() + }) + + test("#given verification attempt is recent and no verification session exists #when pending verification is handled #then handler returns early", async () => { + // given + const promptCalls: string[] = [] + const state = createState(NOW_MS - 1_000) + const { loopState, clearVerificationState, incrementIteration } = createLoopStateController(state) + + // when + await runPendingVerification(state, loopState, promptCalls) + + // then + expect(promptCalls).toHaveLength(0) + expect(clearVerificationState).not.toHaveBeenCalled() + expect(incrementIteration).not.toHaveBeenCalled() + }) + + test("#given verification attempt is older than stuck timeout and no verification session exists #when pending verification is handled #then handler proceeds to failed verification recovery", async () => { + // given + const promptCalls: string[] = [] + const state = createState(NOW_MS - STUCK_VERIFICATION_TIMEOUT_MS - 1) + const { loopState, clearVerificationState, incrementIteration } = createLoopStateController(state) + + // when + await runPendingVerification(state, loopState, promptCalls) + + // then + expect(promptCalls).toHaveLength(1) + expect(clearVerificationState).toHaveBeenCalledTimes(1) + expect(incrementIteration).toHaveBeenCalledTimes(1) + }) + + test("#given legacy verification attempt has no start timestamp and no verification session exists #when pending verification is handled #then handler returns early", async () => { + // given + const promptCalls: string[] = [] + const state = createState() + const { loopState, clearVerificationState, incrementIteration } = createLoopStateController(state) + + // when + await runPendingVerification(state, loopState, promptCalls) + + // then + expect(promptCalls).toHaveLength(0) + expect(clearVerificationState).not.toHaveBeenCalled() + expect(incrementIteration).not.toHaveBeenCalled() + }) +}) diff --git a/src/hooks/ralph-loop/types.ts b/src/hooks/ralph-loop/types.ts index 8c0106b05..00b887439 100644 --- a/src/hooks/ralph-loop/types.ts +++ b/src/hooks/ralph-loop/types.ts @@ -8,6 +8,7 @@ export interface RalphLoopState { completion_promise: string initial_completion_promise?: string verification_attempt_id?: string + verification_attempt_started_at?: number verification_session_id?: string started_at: string prompt: string