fix(atlas): retry boulder after runtime errors
This commit is contained in:
@@ -1,5 +1,4 @@
|
|||||||
import type { PluginInput } from "@opencode-ai/plugin"
|
import type { PluginInput } from "@opencode-ai/plugin"
|
||||||
import type { BackgroundManager } from "../../features/background-agent"
|
|
||||||
import {
|
import {
|
||||||
isAgentRegistered,
|
isAgentRegistered,
|
||||||
resolveRegisteredAgentName,
|
resolveRegisteredAgentName,
|
||||||
@@ -9,7 +8,7 @@ import { createInternalAgentTextPart, resolveInheritedPromptTools } from "../../
|
|||||||
import { HOOK_NAME } from "./hook-name"
|
import { HOOK_NAME } from "./hook-name"
|
||||||
import { BOULDER_CONTINUATION_PROMPT } from "./system-reminder-templates"
|
import { BOULDER_CONTINUATION_PROMPT } from "./system-reminder-templates"
|
||||||
import { resolveRecentPromptContextForSession } from "./recent-model-resolver"
|
import { resolveRecentPromptContextForSession } from "./recent-model-resolver"
|
||||||
import type { SessionState } from "./types"
|
import type { BackgroundTaskStatusProvider, SessionState } from "./types"
|
||||||
|
|
||||||
export type BoulderContinuationResult = "injected" | "skipped_background_tasks" | "skipped_agent_unavailable" | "failed"
|
export type BoulderContinuationResult = "injected" | "skipped_background_tasks" | "skipped_agent_unavailable" | "failed"
|
||||||
|
|
||||||
@@ -25,7 +24,7 @@ export async function injectBoulderContinuation(input: {
|
|||||||
worktreePath?: string
|
worktreePath?: string
|
||||||
preferredTaskSessionId?: string
|
preferredTaskSessionId?: string
|
||||||
preferredTaskTitle?: string
|
preferredTaskTitle?: string
|
||||||
backgroundManager?: BackgroundManager
|
backgroundManager?: BackgroundTaskStatusProvider
|
||||||
sessionState: SessionState
|
sessionState: SessionState
|
||||||
}): Promise<BoulderContinuationResult> {
|
}): Promise<BoulderContinuationResult> {
|
||||||
const {
|
const {
|
||||||
|
|||||||
@@ -25,6 +25,16 @@ export function createAtlasEventHandler(input: {
|
|||||||
state.lastEventWasAbortError = isAbort
|
state.lastEventWasAbortError = isAbort
|
||||||
|
|
||||||
log(`[${HOOK_NAME}] session.error`, { sessionID, isAbort })
|
log(`[${HOOK_NAME}] session.error`, { sessionID, isAbort })
|
||||||
|
if (!isAbort) {
|
||||||
|
const previousInjectedAt = state.lastContinuationInjectedAt
|
||||||
|
await handleAtlasSessionIdle({ ctx, options, getState, sessionID })
|
||||||
|
if (
|
||||||
|
state.lastContinuationInjectedAt !== undefined
|
||||||
|
&& state.lastContinuationInjectedAt !== previousInjectedAt
|
||||||
|
) {
|
||||||
|
state.skipNextIdleAfterRuntimeErrorRetry = true
|
||||||
|
}
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -44,6 +54,7 @@ export function createAtlasEventHandler(input: {
|
|||||||
const state = sessions.get(sessionID)
|
const state = sessions.get(sessionID)
|
||||||
if (state) {
|
if (state) {
|
||||||
state.lastEventWasAbortError = false
|
state.lastEventWasAbortError = false
|
||||||
|
state.skipNextIdleAfterRuntimeErrorRetry = false
|
||||||
if (role === "user") {
|
if (role === "user") {
|
||||||
state.waitingForFinalWaveApproval = false
|
state.waitingForFinalWaveApproval = false
|
||||||
}
|
}
|
||||||
@@ -60,6 +71,7 @@ export function createAtlasEventHandler(input: {
|
|||||||
const state = sessions.get(sessionID)
|
const state = sessions.get(sessionID)
|
||||||
if (state) {
|
if (state) {
|
||||||
state.lastEventWasAbortError = false
|
state.lastEventWasAbortError = false
|
||||||
|
state.skipNextIdleAfterRuntimeErrorRetry = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
@@ -71,6 +83,7 @@ export function createAtlasEventHandler(input: {
|
|||||||
const state = sessions.get(sessionID)
|
const state = sessions.get(sessionID)
|
||||||
if (state) {
|
if (state) {
|
||||||
state.lastEventWasAbortError = false
|
state.lastEventWasAbortError = false
|
||||||
|
state.skipNextIdleAfterRuntimeErrorRetry = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -254,6 +254,12 @@ export async function handleAtlasSessionIdle(input: {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (sessionState.skipNextIdleAfterRuntimeErrorRetry) {
|
||||||
|
sessionState.skipNextIdleAfterRuntimeErrorRetry = false
|
||||||
|
log(`[${HOOK_NAME}] Skipped: stale idle after runtime error retry`, { sessionID })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if (sessionState.promptFailureCount >= MAX_CONSECUTIVE_PROMPT_FAILURES) {
|
if (sessionState.promptFailureCount >= MAX_CONSECUTIVE_PROMPT_FAILURES) {
|
||||||
const timeSinceLastFailure =
|
const timeSinceLastFailure =
|
||||||
sessionState.lastFailureAt !== undefined ? now - sessionState.lastFailureAt : Number.POSITIVE_INFINITY
|
sessionState.lastFailureAt !== undefined ? now - sessionState.lastFailureAt : Number.POSITIVE_INFINITY
|
||||||
|
|||||||
+155
-16
@@ -66,7 +66,7 @@ describe("atlas hook", () => {
|
|||||||
},
|
},
|
||||||
_promptMock: promptMock,
|
_promptMock: promptMock,
|
||||||
_sessionGetMock: sessionGetMock,
|
_sessionGetMock: sessionGetMock,
|
||||||
} as unknown as Parameters<typeof createAtlasHook>[0] & {
|
} as Parameters<typeof createAtlasHook>[0] & {
|
||||||
_promptMock: ReturnType<typeof mock>
|
_promptMock: ReturnType<typeof mock>
|
||||||
_sessionGetMock: ReturnType<typeof mock>
|
_sessionGetMock: ReturnType<typeof mock>
|
||||||
}
|
}
|
||||||
@@ -122,7 +122,7 @@ describe("atlas hook", () => {
|
|||||||
// when - calling with undefined output
|
// when - calling with undefined output
|
||||||
const result = await hook["tool.execute.after"](
|
const result = await hook["tool.execute.after"](
|
||||||
{ tool: "task", sessionID: "session-123" },
|
{ tool: "task", sessionID: "session-123" },
|
||||||
undefined as unknown as { title: string; output: string; metadata: Record<string, unknown> }
|
undefined
|
||||||
)
|
)
|
||||||
|
|
||||||
// then - returns undefined without throwing
|
// then - returns undefined without throwing
|
||||||
@@ -1531,6 +1531,142 @@ session_id: ses_untrusted_999
|
|||||||
expect(mockInput._promptMock).not.toHaveBeenCalled()
|
expect(mockInput._promptMock).not.toHaveBeenCalled()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test("#given boulder has incomplete tasks #when non-abort session error fires #then continuation injects immediately", async () => {
|
||||||
|
// given - boulder state with incomplete plan
|
||||||
|
const planPath = join(TEST_DIR, "test-plan.md")
|
||||||
|
writeFileSync(planPath, "# Plan\n- [ ] Task 1\n- [ ] Task 2")
|
||||||
|
|
||||||
|
const state: BoulderState = {
|
||||||
|
active_plan: planPath,
|
||||||
|
started_at: "2026-01-02T10:00:00Z",
|
||||||
|
session_ids: [MAIN_SESSION_ID],
|
||||||
|
plan_name: "test-plan",
|
||||||
|
}
|
||||||
|
writeBoulderState(TEST_DIR, state)
|
||||||
|
|
||||||
|
const mockInput = createMockPluginInput()
|
||||||
|
const hook = createAtlasHook(mockInput)
|
||||||
|
|
||||||
|
// when - a recoverable runtime error fires without waiting for idle
|
||||||
|
await hook.handler({
|
||||||
|
event: {
|
||||||
|
type: "session.error",
|
||||||
|
properties: {
|
||||||
|
sessionID: MAIN_SESSION_ID,
|
||||||
|
error: { name: "RuntimeError", message: "provider overloaded" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
// then - boulder resumes work immediately
|
||||||
|
expect(mockInput._promptMock).toHaveBeenCalledTimes(1)
|
||||||
|
const callArgs = mockInput._promptMock.mock.calls[0][0]
|
||||||
|
expect(callArgs.path.id).toBe(MAIN_SESSION_ID)
|
||||||
|
expect(callArgs.body.parts[0].text).toContain("incomplete tasks")
|
||||||
|
expect(callArgs.body.parts[0].text).toContain("2 remaining")
|
||||||
|
})
|
||||||
|
|
||||||
|
test("#given boulder retried a runtime error #when stale idle follows #then no delayed duplicate retry is scheduled", async () => {
|
||||||
|
// given - boulder state with incomplete plan
|
||||||
|
const planPath = join(TEST_DIR, "test-plan.md")
|
||||||
|
writeFileSync(planPath, "# Plan\n- [ ] Task 1\n- [ ] Task 2")
|
||||||
|
|
||||||
|
const state: BoulderState = {
|
||||||
|
active_plan: planPath,
|
||||||
|
started_at: "2026-01-02T10:00:00Z",
|
||||||
|
session_ids: [MAIN_SESSION_ID],
|
||||||
|
plan_name: "test-plan",
|
||||||
|
}
|
||||||
|
writeBoulderState(TEST_DIR, state)
|
||||||
|
|
||||||
|
const originalSetTimeout = globalThis.setTimeout
|
||||||
|
const scheduledDelays: number[] = []
|
||||||
|
globalThis.setTimeout = ((_handler: TimerHandler, timeout?: number, ..._args: unknown[]) => {
|
||||||
|
scheduledDelays.push(timeout ?? 0)
|
||||||
|
return originalSetTimeout(() => undefined, 0)
|
||||||
|
}) as typeof setTimeout
|
||||||
|
|
||||||
|
try {
|
||||||
|
const mockInput = createMockPluginInput()
|
||||||
|
const hook = createAtlasHook(mockInput)
|
||||||
|
|
||||||
|
// when - runtime error resumes immediately and OpenCode later emits stale idle
|
||||||
|
await hook.handler({
|
||||||
|
event: {
|
||||||
|
type: "session.error",
|
||||||
|
properties: {
|
||||||
|
sessionID: MAIN_SESSION_ID,
|
||||||
|
error: { name: "RuntimeError", message: "provider overloaded" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
await hook.handler({
|
||||||
|
event: {
|
||||||
|
type: "session.idle",
|
||||||
|
properties: { sessionID: MAIN_SESSION_ID },
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
// then - stale idle is consumed, not converted into another scheduled continuation
|
||||||
|
expect(mockInput._promptMock).toHaveBeenCalledTimes(1)
|
||||||
|
expect(scheduledDelays).toHaveLength(0)
|
||||||
|
} finally {
|
||||||
|
globalThis.setTimeout = originalSetTimeout
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test("#given boulder retried a runtime error #when assistant activity arrives #then next idle can continue", async () => {
|
||||||
|
// given - boulder state with incomplete plan
|
||||||
|
const planPath = join(TEST_DIR, "test-plan.md")
|
||||||
|
writeFileSync(planPath, "# Plan\n- [ ] Task 1\n- [ ] Task 2")
|
||||||
|
|
||||||
|
const state: BoulderState = {
|
||||||
|
active_plan: planPath,
|
||||||
|
started_at: "2026-01-02T10:00:00Z",
|
||||||
|
session_ids: [MAIN_SESSION_ID],
|
||||||
|
plan_name: "test-plan",
|
||||||
|
}
|
||||||
|
writeBoulderState(TEST_DIR, state)
|
||||||
|
|
||||||
|
const originalDateNow = Date.now
|
||||||
|
let now = 1000
|
||||||
|
Date.now = () => now
|
||||||
|
|
||||||
|
try {
|
||||||
|
const mockInput = createMockPluginInput()
|
||||||
|
const hook = createAtlasHook(mockInput)
|
||||||
|
|
||||||
|
// when - runtime error resumes immediately and then the retry run emits assistant activity
|
||||||
|
await hook.handler({
|
||||||
|
event: {
|
||||||
|
type: "session.error",
|
||||||
|
properties: {
|
||||||
|
sessionID: MAIN_SESSION_ID,
|
||||||
|
error: { name: "RuntimeError", message: "provider overloaded" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
await hook.handler({
|
||||||
|
event: {
|
||||||
|
type: "message.updated",
|
||||||
|
properties: { info: { sessionID: MAIN_SESSION_ID, role: "assistant" } },
|
||||||
|
},
|
||||||
|
})
|
||||||
|
now = 7000
|
||||||
|
await hook.handler({
|
||||||
|
event: {
|
||||||
|
type: "session.idle",
|
||||||
|
properties: { sessionID: MAIN_SESSION_ID },
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
// then - assistant activity marks the following idle as real work completion
|
||||||
|
expect(mockInput._promptMock).toHaveBeenCalledTimes(2)
|
||||||
|
} finally {
|
||||||
|
Date.now = originalDateNow
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
test("should skip when background tasks are running", async () => {
|
test("should skip when background tasks are running", async () => {
|
||||||
// given - boulder state with incomplete plan
|
// given - boulder state with incomplete plan
|
||||||
const planPath = join(TEST_DIR, "test-plan.md")
|
const planPath = join(TEST_DIR, "test-plan.md")
|
||||||
@@ -1551,7 +1687,7 @@ session_id: ses_untrusted_999
|
|||||||
const mockInput = createMockPluginInput()
|
const mockInput = createMockPluginInput()
|
||||||
const hook = createAtlasHook(mockInput, {
|
const hook = createAtlasHook(mockInput, {
|
||||||
directory: TEST_DIR,
|
directory: TEST_DIR,
|
||||||
backgroundManager: mockBackgroundManager as any,
|
backgroundManager: mockBackgroundManager,
|
||||||
})
|
})
|
||||||
|
|
||||||
// when
|
// when
|
||||||
@@ -2223,8 +2359,7 @@ session_id: ses_untrusted_999
|
|||||||
})
|
})
|
||||||
|
|
||||||
describe("delayed retry timer (abort-stuck fix)", () => {
|
describe("delayed retry timer (abort-stuck fix)", () => {
|
||||||
const capturedTimers = new Map<number, { callback: Function; cleared: boolean }>()
|
const capturedTimers = new Map<ReturnType<typeof setTimeout>, { callback: () => void | Promise<void>; cleared: boolean }>()
|
||||||
let nextFakeId = 99000
|
|
||||||
const originalSetTimeout = globalThis.setTimeout
|
const originalSetTimeout = globalThis.setTimeout
|
||||||
const originalClearTimeout = globalThis.clearTimeout
|
const originalClearTimeout = globalThis.clearTimeout
|
||||||
const originalDateNow = Date.now
|
const originalDateNow = Date.now
|
||||||
@@ -2232,28 +2367,32 @@ session_id: ses_untrusted_999
|
|||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
capturedTimers.clear()
|
capturedTimers.clear()
|
||||||
nextFakeId = 99000
|
|
||||||
fakeNow = 10000
|
fakeNow = 10000
|
||||||
Date.now = () => fakeNow
|
Date.now = () => fakeNow
|
||||||
|
|
||||||
globalThis.setTimeout = ((callback: Function, delay?: number, ...args: unknown[]) => {
|
globalThis.setTimeout = ((callback: TimerHandler, delay?: number, ...args: unknown[]) => {
|
||||||
const normalized = typeof delay === "number" ? delay : 0
|
const normalized = typeof delay === "number" ? delay : 0
|
||||||
if (normalized >= 5000) {
|
if (normalized >= 5000) {
|
||||||
const id = nextFakeId++
|
const timerID = originalSetTimeout(() => undefined, 0)
|
||||||
capturedTimers.set(id, { callback: () => callback(...args), cleared: false })
|
const capturedCallback = typeof callback === "function"
|
||||||
return id as unknown as ReturnType<typeof setTimeout>
|
? () => callback(...args)
|
||||||
|
: () => undefined
|
||||||
|
capturedTimers.set(timerID, { callback: capturedCallback, cleared: false })
|
||||||
|
return timerID
|
||||||
}
|
}
|
||||||
return originalSetTimeout(callback as Parameters<typeof originalSetTimeout>[0], delay)
|
return typeof callback === "function"
|
||||||
}) as unknown as typeof setTimeout
|
? originalSetTimeout(callback, delay, ...args)
|
||||||
|
: originalSetTimeout(() => undefined, delay)
|
||||||
|
}) as typeof setTimeout
|
||||||
|
|
||||||
globalThis.clearTimeout = ((id?: number | ReturnType<typeof setTimeout>) => {
|
globalThis.clearTimeout = ((id?: ReturnType<typeof setTimeout>) => {
|
||||||
if (typeof id === "number" && capturedTimers.has(id)) {
|
if (id && capturedTimers.has(id)) {
|
||||||
capturedTimers.get(id)!.cleared = true
|
capturedTimers.get(id)!.cleared = true
|
||||||
capturedTimers.delete(id)
|
capturedTimers.delete(id)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
originalClearTimeout(id as Parameters<typeof originalClearTimeout>[0])
|
originalClearTimeout(id)
|
||||||
}) as unknown as typeof clearTimeout
|
}) as typeof clearTimeout
|
||||||
})
|
})
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ export function createToolExecuteAfterHandler(input: {
|
|||||||
pendingTaskRefs: Map<string, PendingTaskRef>
|
pendingTaskRefs: Map<string, PendingTaskRef>
|
||||||
autoCommit: boolean
|
autoCommit: boolean
|
||||||
getState: (sessionID: string) => SessionState
|
getState: (sessionID: string) => SessionState
|
||||||
}): (toolInput: ToolExecuteAfterInput, toolOutput: ToolExecuteAfterOutput) => Promise<void> {
|
}): (toolInput: ToolExecuteAfterInput, toolOutput: ToolExecuteAfterOutput | undefined) => Promise<void> {
|
||||||
const { ctx, pendingFilePaths, pendingTaskRefs, autoCommit, getState } = input
|
const { ctx, pendingFilePaths, pendingTaskRefs, autoCommit, getState } = input
|
||||||
return async (toolInput, toolOutput): Promise<void> => {
|
return async (toolInput, toolOutput): Promise<void> => {
|
||||||
// Guard against undefined output (e.g., from /review command - see issue #1035)
|
// Guard against undefined output (e.g., from /review command - see issue #1035)
|
||||||
|
|||||||
@@ -1,12 +1,15 @@
|
|||||||
import type { AgentOverrides } from "../../config"
|
import type { AgentOverrides } from "../../config"
|
||||||
import type { BackgroundManager } from "../../features/background-agent"
|
|
||||||
import type { TopLevelTaskRef } from "../../features/boulder-state"
|
import type { TopLevelTaskRef } from "../../features/boulder-state"
|
||||||
|
|
||||||
export type ModelInfo = { providerID: string; modelID: string; variant?: string }
|
export type ModelInfo = { providerID: string; modelID: string; variant?: string }
|
||||||
|
|
||||||
|
export interface BackgroundTaskStatusProvider {
|
||||||
|
getTasksByParentSession: (sessionID: string) => Array<{ status: string }>
|
||||||
|
}
|
||||||
|
|
||||||
export interface AtlasHookOptions {
|
export interface AtlasHookOptions {
|
||||||
directory: string
|
directory: string
|
||||||
backgroundManager?: BackgroundManager
|
backgroundManager?: BackgroundTaskStatusProvider
|
||||||
isContinuationStopped?: (sessionID: string) => boolean
|
isContinuationStopped?: (sessionID: string) => boolean
|
||||||
agentOverrides?: AgentOverrides
|
agentOverrides?: AgentOverrides
|
||||||
/** Enable auto-commit after each atomic task completion (default: true) */
|
/** Enable auto-commit after each atomic task completion (default: true) */
|
||||||
@@ -34,6 +37,7 @@ export type PendingTaskRef =
|
|||||||
|
|
||||||
export interface SessionState {
|
export interface SessionState {
|
||||||
lastEventWasAbortError?: boolean
|
lastEventWasAbortError?: boolean
|
||||||
|
skipNextIdleAfterRuntimeErrorRetry?: boolean
|
||||||
lastContinuationInjectedAt?: number
|
lastContinuationInjectedAt?: number
|
||||||
isInjectingContinuation?: boolean
|
isInjectingContinuation?: boolean
|
||||||
promptFailureCount: number
|
promptFailureCount: number
|
||||||
|
|||||||
Reference in New Issue
Block a user