fix(#2732): detect crashed subagent sessions with shorter timeout
When a subagent session disappears from the status registry (process crashed), the main agent was waiting the full stale timeout before acting. Fix: - Add sessionGoneTimeoutMs config option (default 60s, vs 30min normal) - task-poller: use shorter timeout when session is gone from status - manager: verify session existence when gone, fail crashed tasks immediately with descriptive error - Add legacy-plugin-toast hook for #2823 migration warnings - Update schema with new config option
This commit is contained in:
@@ -10,6 +10,7 @@ export const DEFAULT_MAX_TOOL_CALLS = 4000
|
||||
export const DEFAULT_CIRCUIT_BREAKER_CONSECUTIVE_THRESHOLD = 20
|
||||
export const DEFAULT_CIRCUIT_BREAKER_ENABLED = true
|
||||
export const MIN_RUNTIME_BEFORE_STALE_MS = 30_000
|
||||
export const DEFAULT_SESSION_GONE_TIMEOUT_MS = 60_000
|
||||
export const MIN_IDLE_TIME_MS = 5000
|
||||
export const POLLING_INTERVAL_MS = 3000
|
||||
export const TASK_CLEANUP_DELAY_MS = 10 * 60 * 1000
|
||||
|
||||
@@ -1787,6 +1787,53 @@ Use \`background_output(task_id="${task.id}")\` to retrieve this result when rea
|
||||
})
|
||||
}
|
||||
|
||||
private async verifySessionExists(sessionID: string): Promise<boolean> {
|
||||
try {
|
||||
const result = await this.client.session.get({ path: { id: sessionID } })
|
||||
return !!result.data
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
private async failCrashedTask(task: BackgroundTask, errorMessage: string): Promise<void> {
|
||||
task.status = "error"
|
||||
task.error = errorMessage
|
||||
task.completedAt = new Date()
|
||||
if (task.rootSessionID) {
|
||||
this.unregisterRootDescendant(task.rootSessionID)
|
||||
}
|
||||
this.taskHistory.record(task.parentSessionID, { id: task.id, sessionID: task.sessionID, agent: task.agent, description: task.description, status: "error", category: task.category, startedAt: task.startedAt, completedAt: task.completedAt })
|
||||
if (task.concurrencyKey) {
|
||||
this.concurrencyManager.release(task.concurrencyKey)
|
||||
task.concurrencyKey = undefined
|
||||
}
|
||||
|
||||
const completionTimer = this.completionTimers.get(task.id)
|
||||
if (completionTimer) {
|
||||
clearTimeout(completionTimer)
|
||||
this.completionTimers.delete(task.id)
|
||||
}
|
||||
const idleTimer = this.idleDeferralTimers.get(task.id)
|
||||
if (idleTimer) {
|
||||
clearTimeout(idleTimer)
|
||||
this.idleDeferralTimers.delete(task.id)
|
||||
}
|
||||
|
||||
this.cleanupPendingByParent(task)
|
||||
this.clearNotificationsForTask(task.id)
|
||||
removeTaskToastTracking(task.id)
|
||||
this.scheduleTaskRemoval(task.id)
|
||||
if (task.sessionID) {
|
||||
SessionCategoryRegistry.remove(task.sessionID)
|
||||
}
|
||||
|
||||
this.markForNotification(task)
|
||||
this.enqueueNotificationForParent(task.parentSessionID, () => this.notifyParentSession(task)).catch(err => {
|
||||
log("[background-agent] Error in notifyParentSession for crashed task:", { taskId: task.id, error: err })
|
||||
})
|
||||
}
|
||||
|
||||
private async pollRunningTasks(): Promise<void> {
|
||||
if (this.pollingInFlight) return
|
||||
this.pollingInFlight = true
|
||||
@@ -1848,11 +1895,20 @@ Use \`background_output(task_id="${task.id}")\` to retrieve this result when rea
|
||||
}
|
||||
|
||||
// Session is idle or no longer in status response (completed/disappeared)
|
||||
const sessionGoneFromStatus = !sessionStatus
|
||||
const completionSource = sessionStatus?.type === "idle"
|
||||
? "polling (idle status)"
|
||||
: "polling (session gone from status)"
|
||||
const hasValidOutput = await this.validateSessionHasOutput(sessionID)
|
||||
if (!hasValidOutput) {
|
||||
if (sessionGoneFromStatus) {
|
||||
const sessionExists = await this.verifySessionExists(sessionID)
|
||||
if (!sessionExists) {
|
||||
log("[background-agent] Session no longer exists (crashed), marking task as error:", task.id)
|
||||
await this.failCrashedTask(task, "Subagent session no longer exists (process likely crashed). The session disappeared without producing any output.")
|
||||
continue
|
||||
}
|
||||
}
|
||||
log("[background-agent] Polling idle/gone but no valid output yet, waiting:", task.id)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import type { OpencodeClient } from "./opencode-client"
|
||||
|
||||
import {
|
||||
DEFAULT_MESSAGE_STALENESS_TIMEOUT_MS,
|
||||
DEFAULT_SESSION_GONE_TIMEOUT_MS,
|
||||
DEFAULT_STALE_TIMEOUT_MS,
|
||||
MIN_RUNTIME_BEFORE_STALE_MS,
|
||||
TERMINAL_TASK_TTL_MS,
|
||||
@@ -109,6 +110,7 @@ export async function checkAndInterruptStaleTasks(args: {
|
||||
onTaskInterrupted = (task) => removeTaskToastTracking(task.id),
|
||||
} = args
|
||||
const staleTimeoutMs = config?.staleTimeoutMs ?? DEFAULT_STALE_TIMEOUT_MS
|
||||
const sessionGoneTimeoutMs = config?.sessionGoneTimeoutMs ?? DEFAULT_SESSION_GONE_TIMEOUT_MS
|
||||
const now = Date.now()
|
||||
|
||||
const messageStalenessMs = config?.messageStalenessTimeoutMs ?? DEFAULT_MESSAGE_STALENESS_TIMEOUT_MS
|
||||
@@ -122,15 +124,18 @@ export async function checkAndInterruptStaleTasks(args: {
|
||||
|
||||
const sessionStatus = sessionStatuses?.[sessionID]?.type
|
||||
const sessionIsRunning = sessionStatus !== undefined && isActiveSessionStatus(sessionStatus)
|
||||
const sessionGone = sessionStatuses !== undefined && sessionStatus === undefined
|
||||
const runtime = now - startedAt.getTime()
|
||||
|
||||
if (!task.progress?.lastUpdate) {
|
||||
if (sessionIsRunning) continue
|
||||
if (runtime <= messageStalenessMs) continue
|
||||
const effectiveTimeout = sessionGone ? sessionGoneTimeoutMs : messageStalenessMs
|
||||
if (runtime <= effectiveTimeout) continue
|
||||
|
||||
const staleMinutes = Math.round(runtime / 60000)
|
||||
const reason = sessionGone ? "session gone from status registry" : "no activity"
|
||||
task.status = "cancelled"
|
||||
task.error = `Stale timeout (no activity for ${staleMinutes}min since start). This is a FINAL cancellation - do NOT create a replacement task. If the timeout is too short, increase 'background_task.staleTimeoutMs' in .opencode/oh-my-opencode.json.`
|
||||
task.error = `Stale timeout (${reason} for ${staleMinutes}min since start). This is a FINAL cancellation - do NOT create a replacement task. If the timeout is too short, increase 'background_task.${sessionGone ? "sessionGoneTimeoutMs" : "staleTimeoutMs"}' in .opencode/oh-my-opencode.json.`
|
||||
task.completedAt = new Date()
|
||||
|
||||
if (task.concurrencyKey) {
|
||||
@@ -156,12 +161,14 @@ export async function checkAndInterruptStaleTasks(args: {
|
||||
if (runtime < MIN_RUNTIME_BEFORE_STALE_MS) continue
|
||||
|
||||
const timeSinceLastUpdate = now - task.progress.lastUpdate.getTime()
|
||||
if (timeSinceLastUpdate <= staleTimeoutMs) continue
|
||||
const effectiveStaleTimeout = sessionGone ? sessionGoneTimeoutMs : staleTimeoutMs
|
||||
if (timeSinceLastUpdate <= effectiveStaleTimeout) continue
|
||||
if (task.status !== "running") continue
|
||||
|
||||
const staleMinutes = Math.round(timeSinceLastUpdate / 60000)
|
||||
const reason = sessionGone ? "session gone from status registry" : "no activity"
|
||||
task.status = "cancelled"
|
||||
task.error = `Stale timeout (no activity for ${staleMinutes}min). This is a FINAL cancellation - do NOT create a replacement task. If the timeout is too short, increase 'background_task.staleTimeoutMs' in .opencode/oh-my-opencode.json.`
|
||||
task.error = `Stale timeout (${reason} for ${staleMinutes}min). This is a FINAL cancellation - do NOT create a replacement task. If the timeout is too short, increase 'background_task.${sessionGone ? "sessionGoneTimeoutMs" : "staleTimeoutMs"}' in .opencode/oh-my-opencode.json.`
|
||||
task.completedAt = new Date()
|
||||
|
||||
if (task.concurrencyKey) {
|
||||
|
||||
Reference in New Issue
Block a user