Merge pull request #1981 from VespianRex/fix/fallback-sync-model-ui
Fix model fallback retries for main, background, and sync subagents + show runtime fallback model in task UI
This commit is contained in:
YeonGyu-Kim
@@ -0,0 +1,54 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { clearSessionModel, setSessionModel } from "../../shared/session-model-state"
|
||||
import { createBeastModeSystemHook, BEAST_MODE_SYSTEM_PROMPT } from "./hook"
|
||||
|
||||
describe("beast-mode-system hook", () => {
|
||||
test("injects beast mode prompt for copilot gpt-4.1", async () => {
|
||||
//#given
|
||||
const sessionID = "ses_beast"
|
||||
setSessionModel(sessionID, { providerID: "github-copilot", modelID: "gpt-4.1" })
|
||||
const hook = createBeastModeSystemHook()
|
||||
const output = { system: [] as string[] }
|
||||
|
||||
//#when
|
||||
await hook["experimental.chat.system.transform"]?.({ sessionID }, output)
|
||||
|
||||
//#then
|
||||
expect(output.system[0]).toContain("Beast Mode")
|
||||
expect(output.system[0]).toContain(BEAST_MODE_SYSTEM_PROMPT.trim().slice(0, 20))
|
||||
|
||||
clearSessionModel(sessionID)
|
||||
})
|
||||
|
||||
test("does not inject for other models", async () => {
|
||||
//#given
|
||||
const sessionID = "ses_no_beast"
|
||||
setSessionModel(sessionID, { providerID: "quotio", modelID: "gpt-5.3-codex" })
|
||||
const hook = createBeastModeSystemHook()
|
||||
const output = { system: [] as string[] }
|
||||
|
||||
//#when
|
||||
await hook["experimental.chat.system.transform"]?.({ sessionID }, output)
|
||||
|
||||
//#then
|
||||
expect(output.system.length).toBe(0)
|
||||
|
||||
clearSessionModel(sessionID)
|
||||
})
|
||||
|
||||
test("avoids duplicate insertion", async () => {
|
||||
//#given
|
||||
const sessionID = "ses_dupe"
|
||||
setSessionModel(sessionID, { providerID: "github-copilot", modelID: "gpt-4.1" })
|
||||
const hook = createBeastModeSystemHook()
|
||||
const output = { system: [BEAST_MODE_SYSTEM_PROMPT] }
|
||||
|
||||
//#when
|
||||
await hook["experimental.chat.system.transform"]?.({ sessionID }, output)
|
||||
|
||||
//#then
|
||||
expect(output.system.length).toBe(1)
|
||||
|
||||
clearSessionModel(sessionID)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,31 @@
|
||||
import { getSessionModel } from "../../shared/session-model-state"
|
||||
|
||||
export const BEAST_MODE_SYSTEM_PROMPT = `Beast Mode (Copilot GPT-4.1)
|
||||
|
||||
You are an autonomous coding agent. Execute the task end-to-end.
|
||||
- Make a brief plan, then act.
|
||||
- Prefer concrete edits and verification over speculation.
|
||||
- Run relevant tests when feasible.
|
||||
- Do not ask the user to perform actions you can do yourself.
|
||||
- If blocked, state exactly what is needed to proceed.
|
||||
- Keep responses concise and actionable.`
|
||||
|
||||
function isBeastModeModel(model: { providerID: string; modelID: string } | undefined): boolean {
|
||||
return model?.providerID === "github-copilot" && model.modelID === "gpt-4.1"
|
||||
}
|
||||
|
||||
export function createBeastModeSystemHook() {
|
||||
return {
|
||||
"experimental.chat.system.transform": async (
|
||||
input: { sessionID: string },
|
||||
output: { system: string[] },
|
||||
): Promise<void> => {
|
||||
const model = getSessionModel(input.sessionID)
|
||||
if (!isBeastModeModel(model)) return
|
||||
|
||||
if (output.system.some((entry) => entry.includes("Beast Mode"))) return
|
||||
|
||||
output.system.unshift(BEAST_MODE_SYSTEM_PROMPT)
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { createBeastModeSystemHook, BEAST_MODE_SYSTEM_PROMPT } from "./hook"
|
||||
+2
-3
@@ -14,6 +14,7 @@ export { createEmptyTaskResponseDetectorHook } from "./empty-task-response-detec
|
||||
export { createAnthropicContextWindowLimitRecoveryHook, type AnthropicContextWindowLimitRecoveryOptions } from "./anthropic-context-window-limit-recovery";
|
||||
|
||||
export { createThinkModeHook } from "./think-mode";
|
||||
export { createModelFallbackHook, setPendingModelFallback, clearPendingModelFallback, type ModelFallbackState } from "./model-fallback/hook";
|
||||
export { createClaudeCodeHooksHook } from "./claude-code-hooks";
|
||||
export { createRulesInjectorHook } from "./rules-injector";
|
||||
export { createBackgroundNotificationHook } from "./background-notification"
|
||||
@@ -31,7 +32,6 @@ export { createNoSisyphusGptHook } from "./no-sisyphus-gpt";
|
||||
export { createNoHephaestusNonGptHook } from "./no-hephaestus-non-gpt";
|
||||
export { createAutoSlashCommandHook } from "./auto-slash-command";
|
||||
export { createEditErrorRecoveryHook } from "./edit-error-recovery";
|
||||
export { createJsonErrorRecoveryHook } from "./json-error-recovery";
|
||||
export { createPrometheusMdOnlyHook } from "./prometheus-md-only";
|
||||
export { createSisyphusJuniorNotepadHook } from "./sisyphus-junior-notepad";
|
||||
export { createTaskResumeInfoHook } from "./task-resume-info";
|
||||
@@ -47,5 +47,4 @@ export { createPreemptiveCompactionHook } from "./preemptive-compaction";
|
||||
export { createTasksTodowriteDisablerHook } from "./tasks-todowrite-disabler";
|
||||
export { createWriteExistingFileGuardHook } from "./write-existing-file-guard";
|
||||
export { createHashlineReadEnhancerHook } from "./hashline-read-enhancer";
|
||||
export { createHashlineEditDiffEnhancerHook } from "./hashline-edit-diff-enhancer";
|
||||
|
||||
export { createBeastModeSystemHook, BEAST_MODE_SYSTEM_PROMPT } from "./beast-mode-system";
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import { beforeEach, describe, expect, test } from "bun:test"
|
||||
|
||||
import {
|
||||
clearPendingModelFallback,
|
||||
createModelFallbackHook,
|
||||
setPendingModelFallback,
|
||||
} from "./hook"
|
||||
|
||||
describe("model fallback hook", () => {
|
||||
beforeEach(() => {
|
||||
clearPendingModelFallback("ses_model_fallback_main")
|
||||
})
|
||||
|
||||
test("applies pending fallback on chat.message by overriding model", async () => {
|
||||
//#given
|
||||
const hook = createModelFallbackHook() as unknown as {
|
||||
"chat.message"?: (
|
||||
input: { sessionID: string },
|
||||
output: { message: Record<string, unknown>; parts: Array<{ type: string; text?: string }> },
|
||||
) => Promise<void>
|
||||
}
|
||||
|
||||
const set = setPendingModelFallback(
|
||||
"ses_model_fallback_main",
|
||||
"Sisyphus (Ultraworker)",
|
||||
"quotio",
|
||||
"claude-opus-4-6-thinking",
|
||||
)
|
||||
expect(set).toBe(true)
|
||||
|
||||
const output = {
|
||||
message: {
|
||||
model: { providerID: "quotio", modelID: "claude-opus-4-6-thinking" },
|
||||
variant: "max",
|
||||
},
|
||||
parts: [{ type: "text", text: "continue" }],
|
||||
}
|
||||
|
||||
//#when
|
||||
await hook["chat.message"]?.(
|
||||
{ sessionID: "ses_model_fallback_main" },
|
||||
output,
|
||||
)
|
||||
|
||||
//#then
|
||||
expect(output.message["model"]).toEqual({
|
||||
providerID: "quotio",
|
||||
modelID: "claude-opus-4-6",
|
||||
})
|
||||
})
|
||||
|
||||
test("preserves fallback progression across repeated session.error retries", async () => {
|
||||
//#given
|
||||
const hook = createModelFallbackHook() as unknown as {
|
||||
"chat.message"?: (
|
||||
input: { sessionID: string },
|
||||
output: { message: Record<string, unknown>; parts: Array<{ type: string; text?: string }> },
|
||||
) => Promise<void>
|
||||
}
|
||||
const sessionID = "ses_model_fallback_main"
|
||||
|
||||
expect(
|
||||
setPendingModelFallback(sessionID, "Sisyphus (Ultraworker)", "quotio", "claude-opus-4-6-thinking"),
|
||||
).toBe(true)
|
||||
|
||||
const firstOutput = {
|
||||
message: {
|
||||
model: { providerID: "quotio", modelID: "claude-opus-4-6-thinking" },
|
||||
variant: "max",
|
||||
},
|
||||
parts: [{ type: "text", text: "continue" }],
|
||||
}
|
||||
|
||||
//#when - first retry is applied
|
||||
await hook["chat.message"]?.({ sessionID }, firstOutput)
|
||||
|
||||
//#then
|
||||
expect(firstOutput.message["model"]).toEqual({
|
||||
providerID: "quotio",
|
||||
modelID: "claude-opus-4-6",
|
||||
})
|
||||
|
||||
//#when - second error re-arms fallback and should advance to next entry
|
||||
expect(
|
||||
setPendingModelFallback(sessionID, "Sisyphus (Ultraworker)", "quotio", "claude-opus-4-6"),
|
||||
).toBe(true)
|
||||
|
||||
const secondOutput = {
|
||||
message: {
|
||||
model: { providerID: "quotio", modelID: "claude-opus-4-6" },
|
||||
},
|
||||
parts: [{ type: "text", text: "continue" }],
|
||||
}
|
||||
await hook["chat.message"]?.({ sessionID }, secondOutput)
|
||||
|
||||
//#then - chain should progress to entry[1], not repeat entry[0]
|
||||
expect(secondOutput.message["model"]).toEqual({
|
||||
providerID: "quotio",
|
||||
modelID: "gpt-5.3-codex",
|
||||
})
|
||||
expect(secondOutput.message["variant"]).toBe("high")
|
||||
})
|
||||
|
||||
test("shows toast when fallback is applied", async () => {
|
||||
//#given
|
||||
const toastCalls: Array<{ title: string; message: string }> = []
|
||||
const hook = createModelFallbackHook({
|
||||
toast: async ({ title, message }) => {
|
||||
toastCalls.push({ title, message })
|
||||
},
|
||||
}) as unknown as {
|
||||
"chat.message"?: (
|
||||
input: { sessionID: string },
|
||||
output: { message: Record<string, unknown>; parts: Array<{ type: string; text?: string }> },
|
||||
) => Promise<void>
|
||||
}
|
||||
|
||||
const set = setPendingModelFallback(
|
||||
"ses_model_fallback_toast",
|
||||
"Sisyphus (Ultraworker)",
|
||||
"quotio",
|
||||
"claude-opus-4-6-thinking",
|
||||
)
|
||||
expect(set).toBe(true)
|
||||
|
||||
const output = {
|
||||
message: {
|
||||
model: { providerID: "quotio", modelID: "claude-opus-4-6-thinking" },
|
||||
variant: "max",
|
||||
},
|
||||
parts: [{ type: "text", text: "continue" }],
|
||||
}
|
||||
|
||||
//#when
|
||||
await hook["chat.message"]?.({ sessionID: "ses_model_fallback_toast" }, output)
|
||||
|
||||
//#then
|
||||
expect(toastCalls.length).toBe(1)
|
||||
expect(toastCalls[0]?.title).toBe("Model fallback")
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,246 @@
|
||||
import type { FallbackEntry } from "../../shared/model-requirements"
|
||||
import { getAgentConfigKey } from "../../shared/agent-display-names"
|
||||
import { AGENT_MODEL_REQUIREMENTS } from "../../shared/model-requirements"
|
||||
import { readConnectedProvidersCache, readProviderModelsCache } from "../../shared/connected-providers-cache"
|
||||
import { selectFallbackProvider } from "../../shared/model-error-classifier"
|
||||
import { log } from "../../shared/logger"
|
||||
import { getTaskToastManager } from "../../features/task-toast-manager"
|
||||
import type { ChatMessageInput, ChatMessageHandlerOutput } from "../../plugin/chat-message"
|
||||
|
||||
type FallbackToast = (input: {
|
||||
title: string
|
||||
message: string
|
||||
variant?: "info" | "success" | "warning" | "error"
|
||||
duration?: number
|
||||
}) => void | Promise<void>
|
||||
|
||||
type FallbackCallback = (input: {
|
||||
sessionID: string
|
||||
providerID: string
|
||||
modelID: string
|
||||
variant?: string
|
||||
}) => void | Promise<void>
|
||||
|
||||
export type ModelFallbackState = {
|
||||
providerID: string
|
||||
modelID: string
|
||||
fallbackChain: FallbackEntry[]
|
||||
attemptCount: number
|
||||
pending: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Map of sessionID -> pending model fallback state
|
||||
* When a model error occurs, we store the fallback info here.
|
||||
* The next chat.message call will use this to switch to the fallback model.
|
||||
*/
|
||||
const pendingModelFallbacks = new Map<string, ModelFallbackState>()
|
||||
const lastToastKey = new Map<string, string>()
|
||||
const sessionFallbackChains = new Map<string, FallbackEntry[]>()
|
||||
|
||||
export function setSessionFallbackChain(sessionID: string, fallbackChain: FallbackEntry[] | undefined): void {
|
||||
if (!sessionID) return
|
||||
if (!fallbackChain || fallbackChain.length === 0) {
|
||||
sessionFallbackChains.delete(sessionID)
|
||||
return
|
||||
}
|
||||
sessionFallbackChains.set(sessionID, fallbackChain)
|
||||
}
|
||||
|
||||
export function clearSessionFallbackChain(sessionID: string): void {
|
||||
sessionFallbackChains.delete(sessionID)
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets a pending model fallback for a session.
|
||||
* Called when a model error is detected in session.error handler.
|
||||
*/
|
||||
export function setPendingModelFallback(
|
||||
sessionID: string,
|
||||
agentName: string,
|
||||
currentProviderID: string,
|
||||
currentModelID: string,
|
||||
): boolean {
|
||||
const agentKey = getAgentConfigKey(agentName)
|
||||
const requirements = AGENT_MODEL_REQUIREMENTS[agentKey]
|
||||
const sessionFallback = sessionFallbackChains.get(sessionID)
|
||||
const fallbackChain = sessionFallback && sessionFallback.length > 0
|
||||
? sessionFallback
|
||||
: requirements?.fallbackChain
|
||||
|
||||
if (!fallbackChain || fallbackChain.length === 0) {
|
||||
log("[model-fallback] No fallback chain for agent: " + agentName + " (key: " + agentKey + ")")
|
||||
return false
|
||||
}
|
||||
|
||||
const existing = pendingModelFallbacks.get(sessionID)
|
||||
|
||||
if (existing) {
|
||||
// Preserve progression across repeated session.error retries in same session.
|
||||
// We only mark the next turn as pending fallback application.
|
||||
existing.providerID = currentProviderID
|
||||
existing.modelID = currentModelID
|
||||
existing.pending = true
|
||||
if (existing.attemptCount >= existing.fallbackChain.length) {
|
||||
log("[model-fallback] Fallback chain exhausted for session: " + sessionID)
|
||||
return false
|
||||
}
|
||||
log("[model-fallback] Re-armed pending fallback for session: " + sessionID)
|
||||
return true
|
||||
}
|
||||
|
||||
const state: ModelFallbackState = {
|
||||
providerID: currentProviderID,
|
||||
modelID: currentModelID,
|
||||
fallbackChain,
|
||||
attemptCount: 0,
|
||||
pending: true,
|
||||
}
|
||||
|
||||
pendingModelFallbacks.set(sessionID, state)
|
||||
log("[model-fallback] Set pending fallback for session: " + sessionID + ", agent: " + agentName)
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the next fallback model for a session.
|
||||
* Increments attemptCount each time called.
|
||||
*/
|
||||
export function getNextFallback(
|
||||
sessionID: string,
|
||||
): { providerID: string; modelID: string; variant?: string } | null {
|
||||
const state = pendingModelFallbacks.get(sessionID)
|
||||
if (!state) return null
|
||||
|
||||
if (!state.pending) return null
|
||||
|
||||
const { fallbackChain } = state
|
||||
|
||||
const providerModelsCache = readProviderModelsCache()
|
||||
const connectedProviders = providerModelsCache?.connected ?? readConnectedProvidersCache()
|
||||
const connectedSet = connectedProviders ? new Set(connectedProviders) : null
|
||||
|
||||
const isReachable = (entry: FallbackEntry): boolean => {
|
||||
if (!connectedSet) return true
|
||||
|
||||
// Gate only on provider connectivity. Provider model lists can be stale/incomplete,
|
||||
// especially after users manually add models to opencode.json.
|
||||
return entry.providers.some((p) => connectedSet.has(p))
|
||||
}
|
||||
|
||||
while (state.attemptCount < fallbackChain.length) {
|
||||
const attemptCount = state.attemptCount
|
||||
const fallback = fallbackChain[attemptCount]
|
||||
state.attemptCount++
|
||||
|
||||
if (!isReachable(fallback)) {
|
||||
log("[model-fallback] Skipping unreachable fallback for session: " + sessionID + ", attempt: " + attemptCount + ", model: " + fallback.model)
|
||||
continue
|
||||
}
|
||||
|
||||
const providerID = selectFallbackProvider(fallback.providers, state.providerID)
|
||||
state.pending = false
|
||||
|
||||
log("[model-fallback] Using fallback for session: " + sessionID + ", attempt: " + attemptCount + ", model: " + fallback.model)
|
||||
|
||||
return {
|
||||
providerID,
|
||||
modelID: fallback.model,
|
||||
variant: fallback.variant,
|
||||
}
|
||||
}
|
||||
|
||||
log("[model-fallback] No more fallbacks for session: " + sessionID)
|
||||
pendingModelFallbacks.delete(sessionID)
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears the pending fallback for a session.
|
||||
* Called after fallback is successfully applied.
|
||||
*/
|
||||
export function clearPendingModelFallback(sessionID: string): void {
|
||||
pendingModelFallbacks.delete(sessionID)
|
||||
lastToastKey.delete(sessionID)
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if there's a pending fallback for a session.
|
||||
*/
|
||||
export function hasPendingModelFallback(sessionID: string): boolean {
|
||||
const state = pendingModelFallbacks.get(sessionID)
|
||||
return state?.pending === true
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the current fallback state for a session (for debugging).
|
||||
*/
|
||||
export function getFallbackState(sessionID: string): ModelFallbackState | undefined {
|
||||
return pendingModelFallbacks.get(sessionID)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a chat.message hook that applies model fallbacks when pending.
|
||||
*/
|
||||
export function createModelFallbackHook(args?: { toast?: FallbackToast; onApplied?: FallbackCallback }) {
|
||||
const toast = args?.toast
|
||||
const onApplied = args?.onApplied
|
||||
|
||||
return {
|
||||
"chat.message": async (
|
||||
input: ChatMessageInput,
|
||||
output: ChatMessageHandlerOutput,
|
||||
): Promise<void> => {
|
||||
const { sessionID } = input
|
||||
if (!sessionID) return
|
||||
|
||||
const fallback = getNextFallback(sessionID)
|
||||
if (!fallback) return
|
||||
|
||||
output.message["model"] = {
|
||||
providerID: fallback.providerID,
|
||||
modelID: fallback.modelID,
|
||||
}
|
||||
if (fallback.variant !== undefined) {
|
||||
output.message["variant"] = fallback.variant
|
||||
} else {
|
||||
delete output.message["variant"]
|
||||
}
|
||||
if (toast) {
|
||||
const key = `${sessionID}:${fallback.providerID}/${fallback.modelID}:${fallback.variant ?? ""}`
|
||||
if (lastToastKey.get(sessionID) !== key) {
|
||||
lastToastKey.set(sessionID, key)
|
||||
const variantLabel = fallback.variant ? ` (${fallback.variant})` : ""
|
||||
await Promise.resolve(
|
||||
toast({
|
||||
title: "Model fallback",
|
||||
message: `Using ${fallback.providerID}/${fallback.modelID}${variantLabel}`,
|
||||
variant: "warning",
|
||||
duration: 5000,
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
if (onApplied) {
|
||||
await Promise.resolve(
|
||||
onApplied({
|
||||
sessionID,
|
||||
providerID: fallback.providerID,
|
||||
modelID: fallback.modelID,
|
||||
variant: fallback.variant,
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
const toastManager = getTaskToastManager()
|
||||
if (toastManager) {
|
||||
const variantLabel = fallback.variant ? ` (${fallback.variant})` : ""
|
||||
toastManager.updateTaskModelBySession(sessionID, {
|
||||
model: `${fallback.providerID}/${fallback.modelID}${variantLabel}`,
|
||||
type: "runtime-fallback",
|
||||
})
|
||||
}
|
||||
log("[model-fallback] Applied fallback model: " + JSON.stringify(fallback))
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user