fix(ralph-loop): time-bound oracle dispatch wait to prevent stall

The double-fire race fix (#4256) had handlePendingVerification return
early when verification_attempt_id was set but verification_session_id
was not, since the oracle dispatch is still in flight. That guard
introduced a permanent-stall failure mode: if tool-execute-after never
runs (oracle session hangs, crashes, OOM-killed, or tmux killed
externally), verification_session_id stays undefined forever and the
ralph-loop never escapes the pending-verification state.

Track verification_attempt_started_at as a state field, clear it on
restart/clear/setVerificationSessionID, and fall through to
handleFailedVerification when the attempt has been pending past
STUCK_VERIFICATION_TIMEOUT_MS (30 minutes). Legacy persisted states
without the timestamp continue to defer (no timeout to evaluate),
matching pre-fix behavior for that edge case.

Closes pre-publish blocker V25.
This commit is contained in:
YeonGyu-Kim
2026-05-25 01:32:49 +09:00
parent 69c955f61f
commit 14b3523af6
5 changed files with 183 additions and 7 deletions
@@ -45,6 +45,7 @@ export function createLoopStateController(options: {
completion_promise: initialCompletionPromise,
initial_completion_promise: initialCompletionPromise,
verification_attempt_id: undefined,
verification_attempt_started_at: undefined,
verification_session_id: undefined,
ultrawork: loopOptions?.ultrawork,
verification_pending: undefined,
@@ -139,6 +140,7 @@ export function createLoopStateController(options: {
state.verification_pending = true
state.completion_promise = ULTRAWORK_VERIFICATION_PROMISE
state.verification_attempt_id = undefined
state.verification_attempt_started_at = undefined
state.verification_session_id = undefined
state.initial_completion_promise ??= DEFAULT_COMPLETION_PROMISE
@@ -156,6 +158,7 @@ export function createLoopStateController(options: {
}
state.verification_session_id = verificationSessionID
state.verification_attempt_started_at = undefined
if (!writeState(directory, state, stateDir)) {
return null
@@ -175,6 +178,7 @@ export function createLoopStateController(options: {
state.completion_promise = state.initial_completion_promise ?? DEFAULT_COMPLETION_PROMISE
state.verification_pending = undefined
state.verification_attempt_id = undefined
state.verification_attempt_started_at = undefined
state.verification_session_id = undefined
if (typeof messageCountAtStart === "number") {
state.message_count_at_start = messageCountAtStart
@@ -197,6 +201,7 @@ export function createLoopStateController(options: {
state.completion_promise = state.initial_completion_promise ?? DEFAULT_COMPLETION_PROMISE
state.verification_pending = undefined
state.verification_attempt_id = undefined
state.verification_attempt_started_at = undefined
state.verification_session_id = undefined
if (typeof messageCountAtStart === "number") {
state.message_count_at_start = messageCountAtStart
@@ -7,6 +7,8 @@ import { handleFailedVerification } from "./verification-failure-handler"
import { withTimeout } from "./with-timeout"
import type { IterationCommitExpectation } from "./types"
export const STUCK_VERIFICATION_TIMEOUT_MS = 30 * 60 * 1000
type OpenCodeSessionMessage = {
info?: { role?: string }
parts?: Array<{ type?: string; text?: string }>
@@ -138,12 +140,25 @@ export async function handlePendingVerification(
}
if (state.verification_attempt_id && !state.verification_session_id) {
log(`[${HOOK_NAME}] Skipped verification failure: oracle dispatch in flight`, {
sessionID,
verificationAttemptId: state.verification_attempt_id,
iteration: state.iteration,
})
return
const startedAt = state.verification_attempt_started_at
const attemptAgeMs = startedAt !== undefined ? Date.now() - startedAt : undefined
const isStuck = attemptAgeMs !== undefined && attemptAgeMs > STUCK_VERIFICATION_TIMEOUT_MS
if (isStuck) {
log(`[${HOOK_NAME}] Stuck oracle dispatch detected, proceeding to failure handler`, {
sessionID,
verificationAttemptId: state.verification_attempt_id,
attemptAgeMs,
iteration: state.iteration,
})
} else {
log(`[${HOOK_NAME}] Skipped verification failure: oracle dispatch in flight`, {
sessionID,
verificationAttemptId: state.verification_attempt_id,
iteration: state.iteration,
})
return
}
}
const restarted = await handleFailedVerification(ctx, {
+18 -1
View File
@@ -41,6 +41,7 @@ export function readState(directory: string, customPath?: string): RalphLoopStat
}
const ultrawork = data.ultrawork === true || data.ultrawork === "true" ? true : undefined
const verificationAttemptStartedAt = Number(data.verification_attempt_started_at)
const maxIterations =
data.max_iterations === undefined || data.max_iterations === ""
? ultrawork
@@ -65,6 +66,12 @@ export function readState(directory: string, customPath?: string): RalphLoopStat
verification_attempt_id: data.verification_attempt_id
? stripQuotes(data.verification_attempt_id)
: undefined,
verification_attempt_started_at:
data.verification_attempt_started_at === undefined || data.verification_attempt_started_at === ""
? undefined
: Number.isFinite(verificationAttemptStartedAt)
? verificationAttemptStartedAt
: undefined,
verification_session_id: data.verification_session_id
? stripQuotes(data.verification_session_id)
: undefined,
@@ -106,9 +113,19 @@ export function writeState(
const initialCompletionPromiseLine = state.initial_completion_promise
? `initial_completion_promise: "${state.initial_completion_promise}"\n`
: ""
const existingState = readState(directory, customPath)
const verificationAttemptStartedAt = state.verification_session_id || !state.verification_attempt_id
? undefined
: state.verification_attempt_started_at
?? (existingState?.verification_attempt_id !== state.verification_attempt_id
? Date.now()
: existingState.verification_attempt_started_at)
const verificationAttemptLine = state.verification_attempt_id
? `verification_attempt_id: "${state.verification_attempt_id}"\n`
: ""
const verificationAttemptStartedAtLine = typeof verificationAttemptStartedAt === "number"
? `verification_attempt_started_at: ${verificationAttemptStartedAt}\n`
: ""
const verificationSessionLine = state.verification_session_id
? `verification_session_id: "${state.verification_session_id}"\n`
: ""
@@ -124,7 +141,7 @@ export function writeState(
active: ${state.active}
iteration: ${state.iteration}
${maxIterationsLine}completion_promise: "${state.completion_promise}"
${initialCompletionPromiseLine}${verificationAttemptLine}${verificationSessionLine}started_at: "${state.started_at}"
${initialCompletionPromiseLine}${verificationAttemptLine}${verificationAttemptStartedAtLine}${verificationSessionLine}started_at: "${state.started_at}"
${sessionIdLine}${ultraworkLine}${verificationPendingLine}${strategyLine}${messageCountAtStartLine}---
${state.prompt}
`
@@ -0,0 +1,138 @@
import type { PluginInput } from "@opencode-ai/plugin"
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"
import { unsafeTestValue } from "../../../test-support/unsafe-test-value"
import { releaseAllPromptAsyncReservationsForTesting } from "../shared/prompt-async-gate"
import { handlePendingVerification, STUCK_VERIFICATION_TIMEOUT_MS } from "./pending-verification-handler"
import type { RalphLoopState } from "./types"
const NOW_MS = 1_800_000_000_000
type PendingVerificationInput = Parameters<typeof handlePendingVerification>[1]
type LoopStateController = PendingVerificationInput["loopState"]
function createState(verificationAttemptStartedAt?: number): RalphLoopState {
const state: RalphLoopState = {
active: true,
iteration: 2,
completion_promise: "<ulw-verification>",
initial_completion_promise: "<promise>DONE</promise>",
started_at: "2026-01-01T00:00:00.000Z",
prompt: "Ship release blockers",
session_id: "session-123",
ultrawork: true,
verification_pending: true,
verification_attempt_id: "attempt-123",
}
if (verificationAttemptStartedAt === undefined) {
return state
}
return {
...state,
verification_attempt_started_at: verificationAttemptStartedAt,
}
}
function createPluginInput(promptCalls: string[]): PluginInput {
return unsafeTestValue<PluginInput>({
client: {
session: {
messages: async () => ({ data: [] }),
promptAsync: async (input: unknown) => {
promptCalls.push(JSON.stringify(input) ?? "")
return {}
},
abort: async () => ({}),
},
tui: {
showToast: async () => ({}),
},
},
directory: "/tmp/ralph-loop-stuck-oracle-test",
})
}
function createLoopStateController(state: RalphLoopState) {
const clearVerificationState = mock<LoopStateController["clearVerificationState"]>(() => state)
const incrementIteration = mock<LoopStateController["incrementIteration"]>(() => state)
const loopState = {
restartAfterFailedVerification: mock<LoopStateController["restartAfterFailedVerification"]>(() => null),
clearVerificationState,
incrementIteration,
clear: mock<LoopStateController["clear"]>(() => true),
setVerificationSessionID: mock<LoopStateController["setVerificationSessionID"]>(() => null),
} satisfies LoopStateController
return { loopState, clearVerificationState, incrementIteration }
}
async function runPendingVerification(state: RalphLoopState, loopState: LoopStateController, promptCalls: string[]) {
await handlePendingVerification(createPluginInput(promptCalls), {
sessionID: "session-123",
state,
matchesParentSession: true,
matchesVerificationSession: false,
loopState,
directory: "/tmp/ralph-loop-stuck-oracle-test",
apiTimeoutMs: 100,
})
}
describe("ralph-loop stuck oracle dispatch recovery", () => {
const realDateNow = Date.now
beforeEach(() => {
Date.now = () => NOW_MS
})
afterEach(() => {
Date.now = realDateNow
releaseAllPromptAsyncReservationsForTesting()
})
test("#given verification attempt is recent and no verification session exists #when pending verification is handled #then handler returns early", async () => {
// given
const promptCalls: string[] = []
const state = createState(NOW_MS - 1_000)
const { loopState, clearVerificationState, incrementIteration } = createLoopStateController(state)
// when
await runPendingVerification(state, loopState, promptCalls)
// then
expect(promptCalls).toHaveLength(0)
expect(clearVerificationState).not.toHaveBeenCalled()
expect(incrementIteration).not.toHaveBeenCalled()
})
test("#given verification attempt is older than stuck timeout and no verification session exists #when pending verification is handled #then handler proceeds to failed verification recovery", async () => {
// given
const promptCalls: string[] = []
const state = createState(NOW_MS - STUCK_VERIFICATION_TIMEOUT_MS - 1)
const { loopState, clearVerificationState, incrementIteration } = createLoopStateController(state)
// when
await runPendingVerification(state, loopState, promptCalls)
// then
expect(promptCalls).toHaveLength(1)
expect(clearVerificationState).toHaveBeenCalledTimes(1)
expect(incrementIteration).toHaveBeenCalledTimes(1)
})
test("#given legacy verification attempt has no start timestamp and no verification session exists #when pending verification is handled #then handler returns early", async () => {
// given
const promptCalls: string[] = []
const state = createState()
const { loopState, clearVerificationState, incrementIteration } = createLoopStateController(state)
// when
await runPendingVerification(state, loopState, promptCalls)
// then
expect(promptCalls).toHaveLength(0)
expect(clearVerificationState).not.toHaveBeenCalled()
expect(incrementIteration).not.toHaveBeenCalled()
})
})
+1
View File
@@ -8,6 +8,7 @@ export interface RalphLoopState {
completion_promise: string
initial_completion_promise?: string
verification_attempt_id?: string
verification_attempt_started_at?: number
verification_session_id?: string
started_at: string
prompt: string