Fix model fallback across main/background/sync agents

This commit is contained in:
VespianRex
2026-02-19 04:41:00 +02:00
parent d8da89fd5b
commit f5f1d1d4c2
44 changed files with 2500 additions and 605 deletions
@@ -3046,6 +3046,164 @@ describe("BackgroundManager.handleEvent - session.error", () => {
manager.shutdown()
})
test("retry path releases current concurrency slot and prefers current provider in fallback entry", async () => {
//#given
const manager = createBackgroundManager()
const concurrencyManager = getConcurrencyManager(manager)
const concurrencyKey = "quotio/claude-opus-4-6-thinking"
await concurrencyManager.acquire(concurrencyKey)
;(manager as unknown as { processKey: (key: string) => Promise<void> }).processKey = async () => {}
const sessionID = "ses_error_retry"
const task = createMockTask({
id: "task-session-error-retry",
sessionID,
parentSessionID: "parent-session",
parentMessageID: "msg-retry",
description: "task that should retry",
agent: "sisyphus",
status: "running",
concurrencyKey,
model: { providerID: "quotio", modelID: "claude-opus-4-6-thinking" },
fallbackChain: [
{ providers: ["quotio"], model: "claude-opus-4-6", variant: "max" },
{ providers: ["quotio"], model: "claude-opus-4-5" },
],
attemptCount: 0,
})
getTaskMap(manager).set(task.id, task)
//#when
manager.handleEvent({
type: "session.error",
properties: {
sessionID,
error: {
name: "UnknownError",
data: {
message:
"Bad Gateway: {\"error\":{\"message\":\"unknown provider for model claude-opus-4-6-thinking\"}}",
},
},
},
})
//#then
expect(task.status).toBe("pending")
expect(task.attemptCount).toBe(1)
expect(task.model).toEqual({
providerID: "quotio",
modelID: "claude-opus-4-6",
variant: "max",
})
expect(task.concurrencyKey).toBeUndefined()
expect(concurrencyManager.getCount(concurrencyKey)).toBe(0)
manager.shutdown()
})
test("retry path triggers on session.status retry events", async () => {
//#given
const manager = createBackgroundManager()
;(manager as unknown as { processKey: (key: string) => Promise<void> }).processKey = async () => {}
const sessionID = "ses_status_retry"
const task = createMockTask({
id: "task-status-retry",
sessionID,
parentSessionID: "parent-session",
parentMessageID: "msg-status",
description: "task that should retry on status",
agent: "sisyphus",
status: "running",
model: { providerID: "quotio", modelID: "claude-opus-4-6-thinking" },
fallbackChain: [
{ providers: ["quotio"], model: "claude-opus-4-6", variant: "max" },
{ providers: ["quotio"], model: "gpt-5.3-codex", variant: "high" },
],
attemptCount: 0,
})
getTaskMap(manager).set(task.id, task)
//#when
manager.handleEvent({
type: "session.status",
properties: {
sessionID,
status: {
type: "retry",
message: "Provider is overloaded",
},
},
})
//#then
expect(task.status).toBe("pending")
expect(task.attemptCount).toBe(1)
expect(task.model).toEqual({
providerID: "quotio",
modelID: "claude-opus-4-6",
variant: "max",
})
manager.shutdown()
})
test("retry path triggers on message.updated assistant error events", async () => {
//#given
const manager = createBackgroundManager()
;(manager as unknown as { processKey: (key: string) => Promise<void> }).processKey = async () => {}
const sessionID = "ses_message_updated_retry"
const task = createMockTask({
id: "task-message-updated-retry",
sessionID,
parentSessionID: "parent-session",
parentMessageID: "msg-message-updated",
description: "task that should retry on message.updated",
agent: "sisyphus",
status: "running",
model: { providerID: "quotio", modelID: "claude-opus-4-6-thinking" },
fallbackChain: [
{ providers: ["quotio"], model: "claude-opus-4-6", variant: "max" },
{ providers: ["quotio"], model: "gpt-5.3-codex", variant: "high" },
],
attemptCount: 0,
})
getTaskMap(manager).set(task.id, task)
//#when
manager.handleEvent({
type: "message.updated",
properties: {
info: {
id: "msg_errored",
sessionID,
role: "assistant",
error: {
name: "UnknownError",
data: {
message:
"Bad Gateway: {\"error\":{\"message\":\"unknown provider for model claude-opus-4-6-thinking\"}}",
},
},
},
},
})
//#then
expect(task.status).toBe("pending")
expect(task.attemptCount).toBe(1)
expect(task.model).toEqual({
providerID: "quotio",
modelID: "claude-opus-4-6",
variant: "max",
})
manager.shutdown()
})
})
describe("BackgroundManager queue processing - error tasks are skipped", () => {
+234 -7
View File
@@ -5,6 +5,7 @@ import type {
LaunchInput,
ResumeInput,
} from "./types"
import type { FallbackEntry } from "../../shared/model-requirements"
import { TaskHistory } from "./task-history"
import {
log,
@@ -12,6 +13,8 @@ import {
normalizePromptTools,
normalizeSDKResponse,
promptWithModelSuggestionRetry,
readConnectedProvidersCache,
readProviderModelsCache,
resolveInheritedPromptTools,
createInternalAgentTextPart,
} from "../../shared"
@@ -19,6 +22,12 @@ import { setSessionTools } from "../../shared/session-tools-store"
import { ConcurrencyManager } from "./concurrency"
import type { BackgroundTaskConfig, TmuxConfig } from "../../config/schema"
import { isInsideTmux } from "../../shared/tmux"
import {
shouldRetryError,
getNextFallback,
hasMoreFallbacks,
selectFallbackProvider,
} from "../../shared/model-error-classifier"
import {
DEFAULT_MESSAGE_STALENESS_TIMEOUT_MS,
DEFAULT_STALE_TIMEOUT_MS,
@@ -156,6 +165,8 @@ export class BackgroundManager {
parentAgent: input.parentAgent,
parentTools: input.parentTools,
model: input.model,
fallbackChain: input.fallbackChain,
attemptCount: 0,
category: input.category,
}
@@ -677,6 +688,27 @@ export class BackgroundManager {
handleEvent(event: Event): void {
const props = event.properties
if (event.type === "message.updated") {
const info = props?.info
if (!info || typeof info !== "object") return
const sessionID = (info as Record<string, unknown>)["sessionID"]
const role = (info as Record<string, unknown>)["role"]
if (typeof sessionID !== "string" || role !== "assistant") return
const task = this.findBySession(sessionID)
if (!task || task.status !== "running") return
const assistantError = (info as Record<string, unknown>)["error"]
if (!assistantError) return
const errorInfo = {
name: this.extractErrorName(assistantError),
message: this.extractErrorMessage(assistantError),
}
this.tryFallbackRetry(task, errorInfo, "message.updated")
}
if (event.type === "message.part.updated" || event.type === "message.part.delta") {
if (!props || typeof props !== "object" || !("sessionID" in props)) return
const partInfo = props as unknown as MessagePartInfo
@@ -773,10 +805,29 @@ export class BackgroundManager {
const task = this.findBySession(sessionID)
if (!task || task.status !== "running") return
const errorObj = props?.error as { name?: string; message?: string } | undefined
const errorName = errorObj?.name
const errorMessage = props ? this.getSessionErrorMessage(props) : undefined
const errorInfo = { name: errorName, message: errorMessage }
if (this.tryFallbackRetry(task, errorInfo, "session.error")) return
// Original error handling (no retry)
const errorMsg = errorMessage ?? "Session error"
const canRetry =
shouldRetryError(errorInfo) &&
!!task.fallbackChain &&
hasMoreFallbacks(task.fallbackChain, task.attemptCount ?? 0)
log("[background-agent] Session error - no retry:", {
taskId: task.id,
errorName,
errorMessage: errorMsg?.slice(0, 100),
hasFallbackChain: !!task.fallbackChain,
canRetry,
})
task.status = "error"
task.error = errorMessage ?? "Session error"
task.error = errorMsg
task.completedAt = new Date()
this.taskHistory.record(task.parentSessionID, { id: task.id, sessionID: task.sessionID, agent: task.agent, description: task.description, status: "error", category: task.category, startedAt: task.startedAt, completedAt: task.completedAt })
@@ -860,6 +911,129 @@ export class BackgroundManager {
}
}
}
if (event.type === "session.status") {
const sessionID = props?.sessionID as string | undefined
const status = props?.status as { type?: string; message?: string } | undefined
if (!sessionID || status?.type !== "retry") return
const task = this.findBySession(sessionID)
if (!task || task.status !== "running") return
const errorMessage = typeof status.message === "string" ? status.message : undefined
const errorInfo = { name: "SessionRetry", message: errorMessage }
this.tryFallbackRetry(task, errorInfo, "session.status")
}
}
private tryFallbackRetry(
task: BackgroundTask,
errorInfo: { name?: string; message?: string },
source: string,
): boolean {
const fallbackChain = task.fallbackChain
const canRetry =
shouldRetryError(errorInfo) &&
fallbackChain &&
fallbackChain.length > 0 &&
hasMoreFallbacks(fallbackChain, task.attemptCount ?? 0)
if (!canRetry) return false
const attemptCount = task.attemptCount ?? 0
const providerModelsCache = readProviderModelsCache()
const connectedProviders = providerModelsCache?.connected ?? readConnectedProvidersCache()
const connectedSet = connectedProviders ? new Set(connectedProviders) : null
const isReachable = (entry: FallbackEntry): boolean => {
if (!connectedSet) return true
// Gate only on provider connectivity. Provider model lists can be stale/incomplete,
// especially after users manually add models to opencode.json.
return entry.providers.some((p) => connectedSet.has(p))
}
let selectedAttemptCount = attemptCount
let nextFallback: FallbackEntry | undefined
while (fallbackChain && selectedAttemptCount < fallbackChain.length) {
const candidate = getNextFallback(fallbackChain, selectedAttemptCount)
if (!candidate) break
selectedAttemptCount++
if (!isReachable(candidate)) {
log("[background-agent] Skipping unreachable fallback:", {
taskId: task.id,
source,
model: candidate.model,
providers: candidate.providers,
})
continue
}
nextFallback = candidate
break
}
if (!nextFallback) return false
const providerID = selectFallbackProvider(
nextFallback.providers,
task.model?.providerID,
)
log("[background-agent] Retryable error, attempting fallback:", {
taskId: task.id,
source,
errorName: errorInfo.name,
errorMessage: errorInfo.message?.slice(0, 100),
attemptCount: selectedAttemptCount,
nextModel: `${providerID}/${nextFallback.model}`,
})
if (task.concurrencyKey) {
this.concurrencyManager.release(task.concurrencyKey)
task.concurrencyKey = undefined
}
if (task.sessionID) {
this.client.session.abort({ path: { id: task.sessionID } }).catch(() => {})
subagentSessions.delete(task.sessionID)
}
const idleTimer = this.idleDeferralTimers.get(task.id)
if (idleTimer) {
clearTimeout(idleTimer)
this.idleDeferralTimers.delete(task.id)
}
task.attemptCount = selectedAttemptCount
task.model = {
providerID,
modelID: nextFallback.model,
variant: nextFallback.variant,
}
task.status = "pending"
task.sessionID = undefined
task.startedAt = undefined
task.queuedAt = new Date()
task.error = undefined
const key = task.model ? `${task.model.providerID}/${task.model.modelID}` : task.agent
const queue = this.queuesByKey.get(key) ?? []
const retryInput: LaunchInput = {
description: task.description,
prompt: task.prompt,
agent: task.agent,
parentSessionID: task.parentSessionID,
parentMessageID: task.parentMessageID,
parentModel: task.parentModel,
parentAgent: task.parentAgent,
parentTools: task.parentTools,
model: task.model,
fallbackChain: task.fallbackChain,
category: task.category,
}
queue.push({ task, input: retryInput })
this.queuesByKey.set(key, queue)
this.processKey(key)
return true
}
markForNotification(task: BackgroundTask): void {
@@ -1273,10 +1447,13 @@ Use \`background_output(task_id="${task.id}")\` to retrieve this result when rea
if (isCompactionAgent(info?.agent)) {
continue
}
if (info?.agent || info?.model || (info?.modelID && info?.providerID)) {
agent = info.agent ?? task.parentAgent
model = info.model ?? (info.providerID && info.modelID ? { providerID: info.providerID, modelID: info.modelID } : undefined)
tools = normalizePromptTools(info.tools) ?? tools
const normalizedTools = this.isRecord(info?.tools)
? normalizePromptTools(info.tools as Record<string, boolean | "allow" | "deny" | "ask">)
: undefined
if (info?.agent || info?.model || (info?.modelID && info?.providerID) || normalizedTools) {
agent = info?.agent ?? task.parentAgent
model = info?.model ?? (info?.providerID && info?.modelID ? { providerID: info.providerID, modelID: info.modelID } : undefined)
tools = normalizedTools ?? tools
break
}
}
@@ -1296,7 +1473,7 @@ Use \`background_output(task_id="${task.id}")\` to retrieve this result when rea
tools = normalizePromptTools(currentMessage?.tools) ?? tools
}
tools = resolveInheritedPromptTools(task.parentSessionID, tools)
const resolvedTools = resolveInheritedPromptTools(task.parentSessionID, tools)
log("[background-agent] notifyParentSession context:", {
taskId: task.id,
@@ -1311,7 +1488,7 @@ Use \`background_output(task_id="${task.id}")\` to retrieve this result when rea
noReply: !allComplete,
...(agent !== undefined ? { agent } : {}),
...(model !== undefined ? { model } : {}),
...(tools ? { tools } : {}),
...(resolvedTools ? { tools: resolvedTools } : {}),
parts: [createInternalAgentTextPart(notification)],
},
})
@@ -1394,6 +1571,46 @@ Use \`background_output(task_id="${task.id}")\` to retrieve this result when rea
return ""
}
private extractErrorName(error: unknown): string | undefined {
if (this.isRecord(error) && typeof error["name"] === "string") return error["name"]
if (error instanceof Error) return error.name
return undefined
}
private extractErrorMessage(error: unknown): string | undefined {
if (!error) return undefined
if (typeof error === "string") return error
if (error instanceof Error) return error.message
if (this.isRecord(error)) {
const dataRaw = error["data"]
const candidates: unknown[] = [
error,
dataRaw,
error["error"],
this.isRecord(dataRaw) ? (dataRaw as Record<string, unknown>)["error"] : undefined,
error["cause"],
]
for (const candidate of candidates) {
if (typeof candidate === "string" && candidate.length > 0) return candidate
if (
this.isRecord(candidate) &&
typeof candidate["message"] === "string" &&
candidate["message"].length > 0
) {
return candidate["message"]
}
}
}
try {
return JSON.stringify(error)
} catch {
return String(error)
}
}
private isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null
}
@@ -1610,6 +1827,16 @@ Use \`background_output(task_id="${task.id}")\` to retrieve this result when rea
// Progress is already tracked via handleEvent(message.part.updated),
// so we skip the expensive session.messages() fetch here.
// Completion will be detected when session transitions to idle.
if (sessionStatus?.type === "retry") {
const retryMessage = typeof (sessionStatus as { message?: string }).message === "string"
? (sessionStatus as { message?: string }).message
: undefined
const errorInfo = { name: "SessionRetry", message: retryMessage }
if (this.tryFallbackRetry(task, errorInfo, "polling:session.status")) {
continue
}
}
log("[background-agent] Session still running, relying on event-based progress:", {
taskId: task.id,
sessionID,
+8
View File
@@ -1,3 +1,5 @@
import type { FallbackEntry } from "../../shared/model-requirements"
export type BackgroundTaskStatus =
| "pending"
| "running"
@@ -31,6 +33,10 @@ export interface BackgroundTask {
progress?: TaskProgress
parentModel?: { providerID: string; modelID: string }
model?: { providerID: string; modelID: string; variant?: string }
/** Fallback chain for runtime retry on model errors */
fallbackChain?: FallbackEntry[]
/** Number of fallback retry attempts made */
attemptCount?: number
/** Active concurrency slot key */
concurrencyKey?: string
/** Persistent key for re-acquiring concurrency on resume */
@@ -60,6 +66,8 @@ export interface LaunchInput {
parentAgent?: string
parentTools?: Record<string, boolean>
model?: { providerID: string; modelID: string; variant?: string }
/** Fallback chain for runtime retry on model errors */
fallbackChain?: FallbackEntry[]
isUnstableAgent?: boolean
skills?: string[]
skillContent?: string
@@ -1,4 +1,5 @@
export const subagentSessions = new Set<string>()
export const syncSubagentSessions = new Set<string>()
let _mainSessionID: string | undefined
@@ -14,6 +15,7 @@ export function getMainSessionID(): string | undefined {
export function _resetForTesting(): void {
_mainSessionID = undefined
subagentSessions.clear()
syncSubagentSessions.clear()
sessionAgentMap.clear()
}
@@ -25,13 +25,13 @@ export function discoverAllSkillsBlocking(dirs: string[], scopes: SkillScope[]):
const { port1, port2 } = new MessageChannel()
const worker = new Worker(new URL("./discover-worker.ts", import.meta.url), {
workerData: { signal }
// workerData is structured-cloned; pass the SharedArrayBuffer and recreate the view in the worker.
workerData: { signalBuffer: signal.buffer },
})
worker.postMessage({ port: port2 }, [port2])
const input: WorkerInput = { dirs, scopes }
port1.postMessage(input)
// Avoid a race where the worker hasn't attached listeners to the MessagePort yet.
worker.postMessage({ port: port2, input }, [port2])
const waitResult = Atomics.wait(signal, 0, 0, TIMEOUT_MS)
@@ -18,25 +18,24 @@ interface WorkerOutputError {
error: { message: string; stack?: string }
}
const { signal } = workerData as { signal: Int32Array }
const { signalBuffer } = workerData as { signalBuffer: SharedArrayBuffer }
const signal = new Int32Array(signalBuffer)
if (!parentPort) {
throw new Error("Worker must be run with parentPort")
}
parentPort.once("message", (data: { port: MessagePort }) => {
const { port } = data
parentPort.once("message", (data: { port: MessagePort; input: WorkerInput }) => {
const { port, input } = data
port.on("message", async (input: WorkerInput) => {
void (async () => {
try {
const results = await Promise.all(
input.dirs.map(dir => discoverSkillsInDirAsync(dir))
)
const results = await Promise.all(input.dirs.map((dir) => discoverSkillsInDirAsync(dir)))
const skills = results.flat()
const output: WorkerOutputSuccess = { ok: true, skills }
port.postMessage(output)
Atomics.store(signal, 0, 1)
Atomics.notify(signal, 0)
@@ -48,10 +47,10 @@ parentPort.once("message", (data: { port: MessagePort }) => {
stack: error instanceof Error ? error.stack : undefined,
},
}
port.postMessage(output)
Atomics.store(signal, 0, 1)
Atomics.notify(signal, 0)
}
})
})()
})
@@ -217,6 +217,27 @@ describe("TaskToastManager", () => {
expect(call.body.message).toContain("(inherited from parent)")
})
test("should display warning when model is runtime fallback", () => {
// given - runtime-fallback indicates a model swap mid-run
const task = {
id: "task_runtime",
description: "Task with runtime fallback model",
agent: "explore",
isBackground: false,
modelInfo: { model: "quotio/oswe-vscode-prime", type: "runtime-fallback" as const },
}
// when - addTask is called
toastManager.addTask(task)
// then - toast should show fallback warning
expect(mockClient.tui.showToast).toHaveBeenCalled()
const call = mockClient.tui.showToast.mock.calls[0][0]
expect(call.body.message).toContain("[FALLBACK]")
expect(call.body.message).toContain("quotio/oswe-vscode-prime")
expect(call.body.message).toContain("(runtime fallback)")
})
test("should not display model info when user-defined", () => {
// given - a task with user-defined model
const task = {
@@ -257,4 +278,32 @@ describe("TaskToastManager", () => {
expect(call.body.message).not.toContain("[FALLBACK] Model:")
})
})
describe("updateTaskModelBySession", () => {
test("updates task model info and shows fallback toast", () => {
// given - task without model info
const task = {
id: "task_update",
sessionID: "ses_update_1",
description: "Task that will fallback",
agent: "explore",
isBackground: false,
}
toastManager.addTask(task)
mockClient.tui.showToast.mockClear()
// when - runtime fallback applied by session
toastManager.updateTaskModelBySession("ses_update_1", {
model: "nvidia/stepfun-ai/step-3.5-flash",
type: "runtime-fallback",
})
// then - new toast shows fallback model
expect(mockClient.tui.showToast).toHaveBeenCalled()
const call = mockClient.tui.showToast.mock.calls[0][0]
expect(call.body.message).toContain("[FALLBACK]")
expect(call.body.message).toContain("nvidia/stepfun-ai/step-3.5-flash")
expect(call.body.message).toContain("(runtime fallback)")
})
})
})
+20 -3
View File
@@ -20,6 +20,7 @@ export class TaskToastManager {
addTask(task: {
id: string
sessionID?: string
description: string
agent: string
isBackground: boolean
@@ -30,6 +31,7 @@ export class TaskToastManager {
}): void {
const trackedTask: TrackedTask = {
id: task.id,
sessionID: task.sessionID,
description: task.description,
agent: task.agent,
status: task.status ?? "running",
@@ -54,6 +56,18 @@ export class TaskToastManager {
}
}
/**
* Update model info for a task by session ID
*/
updateTaskModelBySession(sessionID: string, modelInfo: ModelFallbackInfo): void {
if (!sessionID) return
const task = Array.from(this.tasks.values()).find((t) => t.sessionID === sessionID)
if (!task) return
if (task.modelInfo?.model === modelInfo.model && task.modelInfo?.type === modelInfo.type) return
task.modelInfo = modelInfo
this.showTaskListToast(task)
}
/**
* Remove completed/error task
*/
@@ -110,14 +124,17 @@ export class TaskToastManager {
const lines: string[] = []
const isFallback = newTask.modelInfo && (
newTask.modelInfo.type === "inherited" || newTask.modelInfo.type === "system-default"
newTask.modelInfo.type === "inherited" ||
newTask.modelInfo.type === "system-default" ||
newTask.modelInfo.type === "runtime-fallback"
)
if (isFallback) {
const suffixMap: Record<"inherited" | "system-default", string> = {
const suffixMap: Record<"inherited" | "system-default" | "runtime-fallback", string> = {
inherited: " (inherited from parent)",
"system-default": " (system default fallback)",
"runtime-fallback": " (runtime fallback)",
}
const suffix = suffixMap[newTask.modelInfo!.type as "inherited" | "system-default"]
const suffix = suffixMap[newTask.modelInfo!.type as "inherited" | "system-default" | "runtime-fallback"]
lines.push(`[FALLBACK] Model: ${newTask.modelInfo!.model}${suffix}`)
lines.push("")
}
+2 -1
View File
@@ -4,12 +4,13 @@ export type TaskStatus = "running" | "queued" | "completed" | "error"
export interface ModelFallbackInfo {
model: string
type: "user-defined" | "inherited" | "category-default" | "system-default"
type: "user-defined" | "inherited" | "category-default" | "system-default" | "runtime-fallback"
source?: ModelSource
}
export interface TrackedTask {
id: string
sessionID?: string
description: string
agent: string
status: TaskStatus