fix(ralph-loop): guard runtime error retries
This commit is contained in:
@@ -144,6 +144,134 @@ describe("ralph-loop non-abort error continuation", () => {
|
|||||||
expect(hook.getState()?.iteration).toBe(2)
|
expect(hook.getState()?.iteration).toBe(2)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test("continues after retry run activity when no stale idle arrived", async () => {
|
||||||
|
// given - an active loop retries a recoverable runtime error
|
||||||
|
const hook = createRalphLoopHook({
|
||||||
|
directory: testDirectory,
|
||||||
|
project: testDirectory,
|
||||||
|
worktree: testDirectory,
|
||||||
|
serverUrl: "http://localhost:4096",
|
||||||
|
$: async () => ({}),
|
||||||
|
client: {
|
||||||
|
session: {
|
||||||
|
messages: async (options: { path: { id: string } }) => {
|
||||||
|
messagesCalls.push({ sessionID: options.path.id })
|
||||||
|
return { data: [] }
|
||||||
|
},
|
||||||
|
promptAsync: async (options: {
|
||||||
|
path: { id: string }
|
||||||
|
body: { parts: Array<{ type: string; text: string }> }
|
||||||
|
}) => {
|
||||||
|
promptCalls.push({
|
||||||
|
sessionID: options.path.id,
|
||||||
|
text: options.body.parts[0]?.text ?? "",
|
||||||
|
})
|
||||||
|
return {}
|
||||||
|
},
|
||||||
|
prompt: async () => ({}),
|
||||||
|
},
|
||||||
|
tui: {
|
||||||
|
showToast: async () => ({}),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as never)
|
||||||
|
|
||||||
|
hook.startLoop("session-123", "Keep working", {
|
||||||
|
messageCountAtStart: 0,
|
||||||
|
maxIterations: 5,
|
||||||
|
})
|
||||||
|
|
||||||
|
await hook.event({
|
||||||
|
event: {
|
||||||
|
type: "session.error",
|
||||||
|
properties: {
|
||||||
|
sessionID: "session-123",
|
||||||
|
error: { name: "RuntimeError" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
// when - the retried run emits real assistant activity before any stale idle
|
||||||
|
await hook.event({
|
||||||
|
event: {
|
||||||
|
type: "message.part.delta",
|
||||||
|
properties: {
|
||||||
|
sessionID: "session-123",
|
||||||
|
messageID: "msg-1",
|
||||||
|
partID: "part-1",
|
||||||
|
field: "text",
|
||||||
|
delta: "working",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
await hook.event({
|
||||||
|
event: { type: "session.idle", properties: { sessionID: "session-123" } },
|
||||||
|
})
|
||||||
|
|
||||||
|
// then - the real idle is allowed to continue the loop
|
||||||
|
expect(promptCalls).toHaveLength(2)
|
||||||
|
expect(hook.getState()?.iteration).toBe(3)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("skips immediate runtime retry while background tasks are running", async () => {
|
||||||
|
// given - an active loop owns running background work
|
||||||
|
const hook = createRalphLoopHook({
|
||||||
|
directory: testDirectory,
|
||||||
|
project: testDirectory,
|
||||||
|
worktree: testDirectory,
|
||||||
|
serverUrl: "http://localhost:4096",
|
||||||
|
$: async () => ({}),
|
||||||
|
client: {
|
||||||
|
session: {
|
||||||
|
messages: async (options: { path: { id: string } }) => {
|
||||||
|
messagesCalls.push({ sessionID: options.path.id })
|
||||||
|
return { data: [] }
|
||||||
|
},
|
||||||
|
promptAsync: async (options: {
|
||||||
|
path: { id: string }
|
||||||
|
body: { parts: Array<{ type: string; text: string }> }
|
||||||
|
}) => {
|
||||||
|
promptCalls.push({
|
||||||
|
sessionID: options.path.id,
|
||||||
|
text: options.body.parts[0]?.text ?? "",
|
||||||
|
})
|
||||||
|
return {}
|
||||||
|
},
|
||||||
|
prompt: async () => ({}),
|
||||||
|
},
|
||||||
|
tui: {
|
||||||
|
showToast: async () => ({}),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as never, {
|
||||||
|
backgroundManager: {
|
||||||
|
getTasksByParentSession: (sessionID: string) => sessionID === "session-123"
|
||||||
|
? [{ status: "running" }]
|
||||||
|
: [],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
hook.startLoop("session-123", "Keep working", {
|
||||||
|
messageCountAtStart: 0,
|
||||||
|
maxIterations: 5,
|
||||||
|
})
|
||||||
|
|
||||||
|
// when - the same session reports a recoverable runtime error
|
||||||
|
await hook.event({
|
||||||
|
event: {
|
||||||
|
type: "session.error",
|
||||||
|
properties: {
|
||||||
|
sessionID: "session-123",
|
||||||
|
error: { name: "RuntimeError" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
// then - Ralph waits for background work instead of starting overlapping continuation
|
||||||
|
expect(promptCalls).toHaveLength(0)
|
||||||
|
expect(hook.getState()?.iteration).toBe(1)
|
||||||
|
})
|
||||||
|
|
||||||
test("stops retrying runtime errors after max iterations", async () => {
|
test("stops retrying runtime errors after max iterations", async () => {
|
||||||
// given - an active Ralph Loop has one retry remaining
|
// given - an active Ralph Loop has one retry remaining
|
||||||
const hook = createRalphLoopHook({
|
const hook = createRalphLoopHook({
|
||||||
|
|||||||
@@ -22,6 +22,47 @@ type LoopStateController = {
|
|||||||
}
|
}
|
||||||
type RalphLoopEventHandlerOptions = { directory: string; apiTimeoutMs: number; getTranscriptPath: (sessionID: string) => string | undefined; checkSessionExists?: RalphLoopOptions["checkSessionExists"]; backgroundManager?: RalphLoopOptions["backgroundManager"]; loopState: LoopStateController }
|
type RalphLoopEventHandlerOptions = { directory: string; apiTimeoutMs: number; getTranscriptPath: (sessionID: string) => string | undefined; checkSessionExists?: RalphLoopOptions["checkSessionExists"]; backgroundManager?: RalphLoopOptions["backgroundManager"]; loopState: LoopStateController }
|
||||||
|
|
||||||
|
function hasRunningBackgroundTasks(
|
||||||
|
backgroundManager: RalphLoopOptions["backgroundManager"],
|
||||||
|
sessionID: string,
|
||||||
|
): boolean {
|
||||||
|
return backgroundManager
|
||||||
|
? backgroundManager.getTasksByParentSession(sessionID).some((task: { status: string }) => task.status === "running")
|
||||||
|
: false
|
||||||
|
}
|
||||||
|
|
||||||
|
function getInfoSessionID(props: Record<string, unknown> | undefined): string | undefined {
|
||||||
|
const info = props?.info as Record<string, unknown> | undefined
|
||||||
|
const sessionID = info?.sessionID
|
||||||
|
return typeof sessionID === "string" ? sessionID : undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
function getRuntimeRetryActivitySessionID(
|
||||||
|
eventType: string,
|
||||||
|
props: Record<string, unknown> | undefined,
|
||||||
|
): string | undefined {
|
||||||
|
if (eventType === "message.updated") {
|
||||||
|
const info = props?.info as Record<string, unknown> | undefined
|
||||||
|
const role = info?.role
|
||||||
|
return role === "assistant" ? getInfoSessionID(props) : undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
if (eventType === "message.part.updated") {
|
||||||
|
if (typeof props?.sessionID === "string") return props.sessionID
|
||||||
|
return getInfoSessionID(props)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (eventType === "message.part.delta") {
|
||||||
|
return typeof props?.sessionID === "string" ? props.sessionID : undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
if (eventType === "tool.execute.before" || eventType === "tool.execute.after") {
|
||||||
|
return typeof props?.sessionID === "string" ? props.sessionID : undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
function isAbortError(error: unknown): boolean {
|
function isAbortError(error: unknown): boolean {
|
||||||
return typeof error === "object"
|
return typeof error === "object"
|
||||||
&& error !== null
|
&& error !== null
|
||||||
@@ -61,6 +102,10 @@ export function createRalphLoopEventHandler(
|
|||||||
|
|
||||||
return async ({ event }: { event: { type: string; properties?: unknown } }): Promise<void> => {
|
return async ({ event }: { event: { type: string; properties?: unknown } }): Promise<void> => {
|
||||||
const props = event.properties as Record<string, unknown> | undefined
|
const props = event.properties as Record<string, unknown> | undefined
|
||||||
|
const runtimeRetryActivitySessionID = getRuntimeRetryActivitySessionID(event.type, props)
|
||||||
|
if (runtimeRetryActivitySessionID) {
|
||||||
|
runtimeErrorRetriedSessions.delete(runtimeRetryActivitySessionID)
|
||||||
|
}
|
||||||
|
|
||||||
if (event.type === "session.idle") {
|
if (event.type === "session.idle") {
|
||||||
const sessionID = props?.sessionID as string | undefined
|
const sessionID = props?.sessionID as string | undefined
|
||||||
@@ -75,18 +120,14 @@ export function createRalphLoopEventHandler(
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
const state = options.loopState.getState()
|
const state = options.loopState.getState()
|
||||||
if (!state || !state.active) {
|
if (!state || !state.active) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
const hasRunningBackgroundTasks = options.backgroundManager
|
if (hasRunningBackgroundTasks(options.backgroundManager, sessionID)) {
|
||||||
? options.backgroundManager.getTasksByParentSession(sessionID).some((task: { status: string }) => task.status === "running")
|
log(`[${HOOK_NAME}] Skipped: background tasks running`, { sessionID })
|
||||||
: false
|
return
|
||||||
|
}
|
||||||
if (hasRunningBackgroundTasks) {
|
|
||||||
log(`[${HOOK_NAME}] Skipped: background tasks running`, { sessionID })
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
const verificationSessionID = state.verification_pending
|
const verificationSessionID = state.verification_pending
|
||||||
? state.verification_session_id
|
? state.verification_session_id
|
||||||
@@ -278,18 +319,23 @@ export function createRalphLoopEventHandler(
|
|||||||
const verificationSessionID = state.verification_pending
|
const verificationSessionID = state.verification_pending
|
||||||
? state.verification_session_id
|
? state.verification_session_id
|
||||||
: undefined
|
: undefined
|
||||||
const matchesParentSession = state.session_id === undefined || state.session_id === sessionID
|
const matchesParentSession = state.session_id === undefined || state.session_id === sessionID
|
||||||
const matchesVerificationSession = verificationSessionID === sessionID
|
const matchesVerificationSession = verificationSessionID === sessionID
|
||||||
if (!matchesParentSession && !matchesVerificationSession) {
|
if (!matchesParentSession && !matchesVerificationSession) {
|
||||||
handleErroredLoopSession(props, options.loopState)
|
handleErroredLoopSession(props, options.loopState)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
log(`[${HOOK_NAME}] Retrying after runtime session error`, {
|
if (hasRunningBackgroundTasks(options.backgroundManager, sessionID)) {
|
||||||
sessionID,
|
log(`[${HOOK_NAME}] Skipped runtime error retry: background tasks running`, { sessionID })
|
||||||
iteration: state.iteration,
|
return
|
||||||
error: String(error),
|
}
|
||||||
})
|
|
||||||
|
log(`[${HOOK_NAME}] Retrying after runtime session error`, {
|
||||||
|
sessionID,
|
||||||
|
iteration: state.iteration,
|
||||||
|
error: String(error),
|
||||||
|
})
|
||||||
|
|
||||||
if (state.verification_pending) {
|
if (state.verification_pending) {
|
||||||
await handlePendingVerification(ctx, {
|
await handlePendingVerification(ctx, {
|
||||||
|
|||||||
Reference in New Issue
Block a user