Fix model fallback across main/background/sync agents

2026-02-19 04:41:00 +02:00
parent d8da89fd5b
commit f5f1d1d4c2
44 changed files with 2500 additions and 605 deletions
@@ -5,6 +5,7 @@ import type {
  LaunchInput,
  ResumeInput,
 } from "./types"
+import type { FallbackEntry } from "../../shared/model-requirements"
 import { TaskHistory } from "./task-history"
 import {
  log,
@@ -12,6 +13,8 @@ import {
  normalizePromptTools,
  normalizeSDKResponse,
  promptWithModelSuggestionRetry,
+  readConnectedProvidersCache,
+  readProviderModelsCache,
  resolveInheritedPromptTools,
  createInternalAgentTextPart,
 } from "../../shared"
@@ -19,6 +22,12 @@ import { setSessionTools } from "../../shared/session-tools-store"
 import { ConcurrencyManager } from "./concurrency"
 import type { BackgroundTaskConfig, TmuxConfig } from "../../config/schema"
 import { isInsideTmux } from "../../shared/tmux"
+import {
+  shouldRetryError,
+  getNextFallback,
+  hasMoreFallbacks,
+  selectFallbackProvider,
+} from "../../shared/model-error-classifier"
 import {
  DEFAULT_MESSAGE_STALENESS_TIMEOUT_MS,
  DEFAULT_STALE_TIMEOUT_MS,
@@ -156,6 +165,8 @@ export class BackgroundManager {
      parentAgent: input.parentAgent,
      parentTools: input.parentTools,
      model: input.model,
+      fallbackChain: input.fallbackChain,
+      attemptCount: 0,
      category: input.category,
    }

@@ -677,6 +688,27 @@ export class BackgroundManager {
  handleEvent(event: Event): void {
    const props = event.properties

+    if (event.type === "message.updated") {
+      const info = props?.info
+      if (!info || typeof info !== "object") return
+
+      const sessionID = (info as Record<string, unknown>)["sessionID"]
+      const role = (info as Record<string, unknown>)["role"]
+      if (typeof sessionID !== "string" || role !== "assistant") return
+
+      const task = this.findBySession(sessionID)
+      if (!task || task.status !== "running") return
+
+      const assistantError = (info as Record<string, unknown>)["error"]
+      if (!assistantError) return
+
+      const errorInfo = {
+        name: this.extractErrorName(assistantError),
+        message: this.extractErrorMessage(assistantError),
+      }
+      this.tryFallbackRetry(task, errorInfo, "message.updated")
+    }
+
    if (event.type === "message.part.updated" || event.type === "message.part.delta") {
      if (!props || typeof props !== "object" || !("sessionID" in props)) return
      const partInfo = props as unknown as MessagePartInfo
@@ -773,10 +805,29 @@ export class BackgroundManager {
      const task = this.findBySession(sessionID)
      if (!task || task.status !== "running") return

+      const errorObj = props?.error as { name?: string; message?: string } | undefined
+      const errorName = errorObj?.name
      const errorMessage = props ? this.getSessionErrorMessage(props) : undefined

+      const errorInfo = { name: errorName, message: errorMessage }
+      if (this.tryFallbackRetry(task, errorInfo, "session.error")) return
+
+      // Original error handling (no retry)
+      const errorMsg = errorMessage ?? "Session error"
+      const canRetry =
+        shouldRetryError(errorInfo) &&
+        !!task.fallbackChain &&
+        hasMoreFallbacks(task.fallbackChain, task.attemptCount ?? 0)
+      log("[background-agent] Session error - no retry:", {
+        taskId: task.id,
+        errorName,
+        errorMessage: errorMsg?.slice(0, 100),
+        hasFallbackChain: !!task.fallbackChain,
+        canRetry,
+      })
+
      task.status = "error"
-      task.error = errorMessage ?? "Session error"
+      task.error = errorMsg
      task.completedAt = new Date()
      this.taskHistory.record(task.parentSessionID, { id: task.id, sessionID: task.sessionID, agent: task.agent, description: task.description, status: "error", category: task.category, startedAt: task.startedAt, completedAt: task.completedAt })

@@ -860,6 +911,129 @@ export class BackgroundManager {
        }
      }
    }
+
+    if (event.type === "session.status") {
+      const sessionID = props?.sessionID as string | undefined
+      const status = props?.status as { type?: string; message?: string } | undefined
+      if (!sessionID || status?.type !== "retry") return
+
+      const task = this.findBySession(sessionID)
+      if (!task || task.status !== "running") return
+
+      const errorMessage = typeof status.message === "string" ? status.message : undefined
+      const errorInfo = { name: "SessionRetry", message: errorMessage }
+      this.tryFallbackRetry(task, errorInfo, "session.status")
+    }
+  }
+
+  private tryFallbackRetry(
+    task: BackgroundTask,
+    errorInfo: { name?: string; message?: string },
+    source: string,
+  ): boolean {
+    const fallbackChain = task.fallbackChain
+    const canRetry =
+      shouldRetryError(errorInfo) &&
+      fallbackChain &&
+      fallbackChain.length > 0 &&
+      hasMoreFallbacks(fallbackChain, task.attemptCount ?? 0)
+
+    if (!canRetry) return false
+
+    const attemptCount = task.attemptCount ?? 0
+    const providerModelsCache = readProviderModelsCache()
+    const connectedProviders = providerModelsCache?.connected ?? readConnectedProvidersCache()
+    const connectedSet = connectedProviders ? new Set(connectedProviders) : null
+
+    const isReachable = (entry: FallbackEntry): boolean => {
+      if (!connectedSet) return true
+
+      // Gate only on provider connectivity. Provider model lists can be stale/incomplete,
+      // especially after users manually add models to opencode.json.
+      return entry.providers.some((p) => connectedSet.has(p))
+    }
+
+    let selectedAttemptCount = attemptCount
+    let nextFallback: FallbackEntry | undefined
+    while (fallbackChain && selectedAttemptCount < fallbackChain.length) {
+      const candidate = getNextFallback(fallbackChain, selectedAttemptCount)
+      if (!candidate) break
+      selectedAttemptCount++
+      if (!isReachable(candidate)) {
+        log("[background-agent] Skipping unreachable fallback:", {
+          taskId: task.id,
+          source,
+          model: candidate.model,
+          providers: candidate.providers,
+        })
+        continue
+      }
+      nextFallback = candidate
+      break
+    }
+    if (!nextFallback) return false
+
+    const providerID = selectFallbackProvider(
+      nextFallback.providers,
+      task.model?.providerID,
+    )
+
+    log("[background-agent] Retryable error, attempting fallback:", {
+      taskId: task.id,
+      source,
+      errorName: errorInfo.name,
+      errorMessage: errorInfo.message?.slice(0, 100),
+      attemptCount: selectedAttemptCount,
+      nextModel: `${providerID}/${nextFallback.model}`,
+    })
+
+    if (task.concurrencyKey) {
+      this.concurrencyManager.release(task.concurrencyKey)
+      task.concurrencyKey = undefined
+    }
+
+    if (task.sessionID) {
+      this.client.session.abort({ path: { id: task.sessionID } }).catch(() => {})
+      subagentSessions.delete(task.sessionID)
+    }
+
+    const idleTimer = this.idleDeferralTimers.get(task.id)
+    if (idleTimer) {
+      clearTimeout(idleTimer)
+      this.idleDeferralTimers.delete(task.id)
+    }
+
+    task.attemptCount = selectedAttemptCount
+    task.model = {
+      providerID,
+      modelID: nextFallback.model,
+      variant: nextFallback.variant,
+    }
+    task.status = "pending"
+    task.sessionID = undefined
+    task.startedAt = undefined
+    task.queuedAt = new Date()
+    task.error = undefined
+
+    const key = task.model ? `${task.model.providerID}/${task.model.modelID}` : task.agent
+    const queue = this.queuesByKey.get(key) ?? []
+    const retryInput: LaunchInput = {
+      description: task.description,
+      prompt: task.prompt,
+      agent: task.agent,
+      parentSessionID: task.parentSessionID,
+      parentMessageID: task.parentMessageID,
+      parentModel: task.parentModel,
+      parentAgent: task.parentAgent,
+      parentTools: task.parentTools,
+      model: task.model,
+      fallbackChain: task.fallbackChain,
+      category: task.category,
+    }
+    queue.push({ task, input: retryInput })
+    this.queuesByKey.set(key, queue)
+    this.processKey(key)
+    return true
  }

  markForNotification(task: BackgroundTask): void {
@@ -1273,10 +1447,13 @@ Use \`background_output(task_id="${task.id}")\` to retrieve this result when rea
            if (isCompactionAgent(info?.agent)) {
              continue
            }
-            if (info?.agent || info?.model || (info?.modelID && info?.providerID)) {
-              agent = info.agent ?? task.parentAgent
-              model = info.model ?? (info.providerID && info.modelID ? { providerID: info.providerID, modelID: info.modelID } : undefined)
-              tools = normalizePromptTools(info.tools) ?? tools
+            const normalizedTools = this.isRecord(info?.tools)
+              ? normalizePromptTools(info.tools as Record<string, boolean | "allow" | "deny" | "ask">)
+              : undefined
+            if (info?.agent || info?.model || (info?.modelID && info?.providerID) || normalizedTools) {
+              agent = info?.agent ?? task.parentAgent
+              model = info?.model ?? (info?.providerID && info?.modelID ? { providerID: info.providerID, modelID: info.modelID } : undefined)
+              tools = normalizedTools ?? tools
              break
            }
          }
@@ -1296,7 +1473,7 @@ Use \`background_output(task_id="${task.id}")\` to retrieve this result when rea
          tools = normalizePromptTools(currentMessage?.tools) ?? tools
        }

-        tools = resolveInheritedPromptTools(task.parentSessionID, tools)
+        const resolvedTools = resolveInheritedPromptTools(task.parentSessionID, tools)

        log("[background-agent] notifyParentSession context:", {
          taskId: task.id,
@@ -1311,7 +1488,7 @@ Use \`background_output(task_id="${task.id}")\` to retrieve this result when rea
              noReply: !allComplete,
              ...(agent !== undefined ? { agent } : {}),
              ...(model !== undefined ? { model } : {}),
-              ...(tools ? { tools } : {}),
+              ...(resolvedTools ? { tools: resolvedTools } : {}),
              parts: [createInternalAgentTextPart(notification)],
            },
          })
@@ -1394,6 +1571,46 @@ Use \`background_output(task_id="${task.id}")\` to retrieve this result when rea
    return ""
  }

+  private extractErrorName(error: unknown): string | undefined {
+    if (this.isRecord(error) && typeof error["name"] === "string") return error["name"]
+    if (error instanceof Error) return error.name
+    return undefined
+  }
+
+  private extractErrorMessage(error: unknown): string | undefined {
+    if (!error) return undefined
+    if (typeof error === "string") return error
+    if (error instanceof Error) return error.message
+
+    if (this.isRecord(error)) {
+      const dataRaw = error["data"]
+      const candidates: unknown[] = [
+        error,
+        dataRaw,
+        error["error"],
+        this.isRecord(dataRaw) ? (dataRaw as Record<string, unknown>)["error"] : undefined,
+        error["cause"],
+      ]
+
+      for (const candidate of candidates) {
+        if (typeof candidate === "string" && candidate.length > 0) return candidate
+        if (
+          this.isRecord(candidate) &&
+          typeof candidate["message"] === "string" &&
+          candidate["message"].length > 0
+        ) {
+          return candidate["message"]
+        }
+      }
+    }
+
+    try {
+      return JSON.stringify(error)
+    } catch {
+      return String(error)
+    }
+  }
+
  private isRecord(value: unknown): value is Record<string, unknown> {
    return typeof value === "object" && value !== null
  }
@@ -1610,6 +1827,16 @@ Use \`background_output(task_id="${task.id}")\` to retrieve this result when rea
        // Progress is already tracked via handleEvent(message.part.updated),
        // so we skip the expensive session.messages() fetch here.
        // Completion will be detected when session transitions to idle.
+        if (sessionStatus?.type === "retry") {
+          const retryMessage = typeof (sessionStatus as { message?: string }).message === "string"
+            ? (sessionStatus as { message?: string }).message
+            : undefined
+          const errorInfo = { name: "SessionRetry", message: retryMessage }
+          if (this.tryFallbackRetry(task, errorInfo, "polling:session.status")) {
+            continue
+          }
+        }
+
        log("[background-agent] Session still running, relying on event-based progress:", {
          taskId: task.id,
          sessionID,