Merge pull request #2302 from RaviTharuma/fix/runtime-fallback-cooldown-session-status

Fix cooldown fallback switching across runtime/model fallback hooks
This commit is contained in:
YeonGyu-Kim
2026-03-09 13:38:46 +09:00
committed by GitHub
29 changed files with 1416 additions and 77 deletions
+116 -1
View File
@@ -140,6 +140,121 @@ describe("model fallback hook", () => {
expect(secondOutput.message["variant"]).toBeUndefined()
})
test("does not re-arm fallback when one is already pending", () => {
//#given
const sessionID = "ses_model_fallback_pending_guard"
clearPendingModelFallback(sessionID)
//#when
const firstSet = setPendingModelFallback(
sessionID,
"Sisyphus (Ultraworker)",
"anthropic",
"claude-opus-4-6-thinking",
)
const secondSet = setPendingModelFallback(
sessionID,
"Sisyphus (Ultraworker)",
"anthropic",
"claude-opus-4-6-thinking",
)
//#then
expect(firstSet).toBe(true)
expect(secondSet).toBe(false)
clearPendingModelFallback(sessionID)
})
test("skips no-op fallback entries that resolve to same provider/model", async () => {
//#given
const sessionID = "ses_model_fallback_noop_skip"
clearPendingModelFallback(sessionID)
const hook = createModelFallbackHook() as unknown as {
"chat.message"?: (
input: { sessionID: string },
output: { message: Record<string, unknown>; parts: Array<{ type: string; text?: string }> },
) => Promise<void>
}
setSessionFallbackChain(sessionID, [
{ providers: ["anthropic"], model: "claude-opus-4-6" },
{ providers: ["opencode"], model: "kimi-k2.5-free" },
])
expect(
setPendingModelFallback(
sessionID,
"Sisyphus (Ultraworker)",
"anthropic",
"claude-opus-4-6",
),
).toBe(true)
const output = {
message: {
model: { providerID: "anthropic", modelID: "claude-opus-4-6" },
},
parts: [{ type: "text", text: "continue" }],
}
//#when
await hook["chat.message"]?.({ sessionID }, output)
//#then
expect(output.message["model"]).toEqual({
providerID: "opencode",
modelID: "kimi-k2.5-free",
})
clearPendingModelFallback(sessionID)
})
test("skips no-op fallback entries even when variant differs", async () => {
//#given
const sessionID = "ses_model_fallback_noop_variant_skip"
clearPendingModelFallback(sessionID)
const hook = createModelFallbackHook() as unknown as {
"chat.message"?: (
input: { sessionID: string },
output: { message: Record<string, unknown>; parts: Array<{ type: string; text?: string }> },
) => Promise<void>
}
setSessionFallbackChain(sessionID, [
{ providers: ["quotio"], model: "claude-opus-4-6", variant: "max" },
{ providers: ["quotio"], model: "gpt-5.2" },
])
expect(
setPendingModelFallback(
sessionID,
"Sisyphus (Ultraworker)",
"quotio",
"claude-opus-4-6",
),
).toBe(true)
const output = {
message: {
model: { providerID: "quotio", modelID: "claude-opus-4-6" },
variant: "max",
},
parts: [{ type: "text", text: "continue" }],
}
//#when
await hook["chat.message"]?.({ sessionID }, output)
//#then
expect(output.message["model"]).toEqual({
providerID: "quotio",
modelID: "gpt-5.2",
})
expect(output.message["variant"]).toBeUndefined()
clearPendingModelFallback(sessionID)
})
test("shows toast when fallback is applied", async () => {
//#given
const toastCalls: Array<{ title: string; message: string }> = []
@@ -199,7 +314,7 @@ describe("model fallback hook", () => {
sessionID,
"Atlas (Plan Executor)",
"github-copilot",
"claude-sonnet-4-6",
"claude-sonnet-4-5",
)
expect(set).toBe(true)
+23 -1
View File
@@ -39,6 +39,12 @@ const pendingModelFallbacks = new Map<string, ModelFallbackState>()
const lastToastKey = new Map<string, string>()
const sessionFallbackChains = new Map<string, FallbackEntry[]>()
function canonicalizeModelID(modelID: string): string {
return modelID
.toLowerCase()
.replace(/\./g, "-")
}
export function setSessionFallbackChain(sessionID: string, fallbackChain: FallbackEntry[] | undefined): void {
if (!sessionID) return
if (!fallbackChain || fallbackChain.length === 0) {
@@ -77,6 +83,11 @@ export function setPendingModelFallback(
const existing = pendingModelFallbacks.get(sessionID)
if (existing) {
if (existing.pending) {
log("[model-fallback] Pending fallback already armed for session: " + sessionID)
return false
}
// Preserve progression across repeated session.error retries in same session.
// We only mark the next turn as pending fallback application.
existing.providerID = currentProviderID
@@ -140,13 +151,24 @@ export function getNextFallback(
}
const providerID = selectFallbackProvider(fallback.providers, state.providerID)
const modelID = transformModelForProvider(providerID, fallback.model)
const isNoOpFallback =
providerID.toLowerCase() === state.providerID.toLowerCase() &&
canonicalizeModelID(modelID) === canonicalizeModelID(state.modelID)
if (isNoOpFallback) {
log("[model-fallback] Skipping no-op fallback for session: " + sessionID + ", attempt: " + attemptCount + ", model: " + fallback.model)
continue
}
state.pending = false
log("[model-fallback] Using fallback for session: " + sessionID + ", attempt: " + attemptCount + ", model: " + fallback.model)
return {
providerID,
modelID: transformModelForProvider(providerID, fallback.model),
modelID,
variant: fallback.variant,
}
}
+4
View File
@@ -26,6 +26,10 @@ export const RETRYABLE_ERROR_PATTERNS = [
/rate.?limit/i,
/too.?many.?requests/i,
/quota.?exceeded/i,
/quota\s+will\s+reset\s+after/i,
/all\s+credentials\s+for\s+model/i,
/cool(?:ing)?\s+down/i,
/exhausted\s+your\s+capacity/i,
/usage\s+limit\s+has\s+been\s+reached/i,
/service.?unavailable/i,
/overloaded/i,
@@ -0,0 +1,60 @@
import { describe, expect, test } from "bun:test"
import { extractAutoRetrySignal, isRetryableError } from "./error-classifier"
describe("runtime-fallback error classifier", () => {
test("detects cooling-down auto-retry status signals", () => {
//#given
const info = {
status:
"All credentials for model claude-opus-4-6-thinking are cooling down [retrying in ~5 days attempt #1]",
}
//#when
const signal = extractAutoRetrySignal(info)
//#then
expect(signal).toBeDefined()
})
test("detects single-word cooldown auto-retry status signals", () => {
//#given
const info = {
status:
"All credentials for model claude-opus-4-6 are cooldown [retrying in 7m 56s attempt #1]",
}
//#when
const signal = extractAutoRetrySignal(info)
//#then
expect(signal).toBeDefined()
})
test("treats cooling-down retry messages as retryable", () => {
//#given
const error = {
message:
"All credentials for model claude-opus-4-6-thinking are cooling down [retrying in ~5 days attempt #1]",
}
//#when
const retryable = isRetryableError(error, [400, 403, 408, 429, 500, 502, 503, 504, 529])
//#then
expect(retryable).toBe(true)
})
test("ignores non-retry assistant status text", () => {
//#given
const info = {
status: "Thinking...",
}
//#when
const signal = extractAutoRetrySignal(info)
//#then
expect(signal).toBeUndefined()
})
})
@@ -102,7 +102,7 @@ export interface AutoRetrySignal {
export const AUTO_RETRY_PATTERNS: Array<(combined: string) => boolean> = [
(combined) => /retrying\s+in/i.test(combined),
(combined) =>
/(?:too\s+many\s+requests|quota\s*exceeded|usage\s+limit|rate\s+limit|limit\s+reached)/i.test(combined),
/(?:too\s+many\s+requests|quota\s*exceeded|quota\s+will\s+reset\s+after|usage\s+limit|rate\s+limit|limit\s+reached|all\s+credentials\s+for\s+model|cool(?:ing)?\s*down|exhausted\s+your\s+capacity)/i.test(combined),
]
export function extractAutoRetrySignal(info: Record<string, unknown> | undefined): AutoRetrySignal | undefined {
+87 -1
View File
@@ -2,13 +2,15 @@ import type { HookDeps } from "./types"
import type { AutoRetryHelpers } from "./auto-retry"
import { HOOK_NAME } from "./constants"
import { log } from "../../shared/logger"
import { extractStatusCode, extractErrorName, classifyErrorType, isRetryableError } from "./error-classifier"
import { extractStatusCode, extractErrorName, classifyErrorType, isRetryableError, extractAutoRetrySignal } from "./error-classifier"
import { createFallbackState, prepareFallback } from "./fallback-state"
import { getFallbackModelsForSession } from "./fallback-models"
import { SessionCategoryRegistry } from "../../shared/session-category-registry"
import { normalizeRetryStatusMessage, extractRetryAttempt } from "../../shared/retry-status-utils"
export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
const { config, pluginConfig, sessionStates, sessionLastAccess, sessionRetryInFlight, sessionAwaitingFallbackResult, sessionFallbackTimeouts } = deps
const sessionStatusRetryKeys = new Map<string, string>()
const handleSessionCreated = (props: Record<string, unknown> | undefined) => {
const sessionInfo = props?.info as { id?: string; model?: string } | undefined
@@ -33,6 +35,7 @@ export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
sessionRetryInFlight.delete(sessionID)
sessionAwaitingFallbackResult.delete(sessionID)
helpers.clearSessionFallbackTimeout(sessionID)
sessionStatusRetryKeys.delete(sessionID)
SessionCategoryRegistry.remove(sessionID)
}
}
@@ -182,6 +185,88 @@ export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
}
}
const handleSessionStatus = async (props: Record<string, unknown> | undefined) => {
const sessionID = props?.sessionID as string | undefined
const status = props?.status as { type?: string; message?: string; attempt?: number } | undefined
const agent = props?.agent as string | undefined
const model = props?.model as string | undefined
if (!sessionID || status?.type !== "retry") return
const retryMessage = typeof status.message === "string" ? status.message : ""
const retrySignal = extractAutoRetrySignal({ status: retryMessage, message: retryMessage })
if (!retrySignal) return
const retryKey = `${extractRetryAttempt(status.attempt, retryMessage)}:${normalizeRetryStatusMessage(retryMessage)}`
if (sessionStatusRetryKeys.get(sessionID) === retryKey) {
return
}
sessionStatusRetryKeys.set(sessionID, retryKey)
if (sessionRetryInFlight.has(sessionID)) {
log(`[${HOOK_NAME}] session.status retry skipped — retry already in flight`, { sessionID })
return
}
const resolvedAgent = await helpers.resolveAgentForSessionFromContext(sessionID, agent)
const fallbackModels = getFallbackModelsForSession(sessionID, resolvedAgent, pluginConfig)
if (fallbackModels.length === 0) return
let state = sessionStates.get(sessionID)
if (!state) {
const detectedAgent = resolvedAgent
const agentConfig = detectedAgent
? pluginConfig?.agents?.[detectedAgent as keyof typeof pluginConfig.agents]
: undefined
const inferredModel = model || (agentConfig?.model as string | undefined)
if (!inferredModel) {
log(`[${HOOK_NAME}] session.status retry missing model info, cannot fallback`, { sessionID })
return
}
state = createFallbackState(inferredModel)
sessionStates.set(sessionID, state)
}
sessionLastAccess.set(sessionID, Date.now())
if (state.pendingFallbackModel) {
log(`[${HOOK_NAME}] session.status retry skipped (pending fallback in progress)`, {
sessionID,
pendingFallbackModel: state.pendingFallbackModel,
})
return
}
log(`[${HOOK_NAME}] Detected provider auto-retry signal in session.status`, {
sessionID,
model: state.currentModel,
retryAttempt: status.attempt,
})
await helpers.abortSessionRequest(sessionID, "session.status.retry-signal")
const result = prepareFallback(sessionID, state, fallbackModels, config)
if (result.success && config.notify_on_fallback) {
await deps.ctx.client.tui
.showToast({
body: {
title: "Model Fallback",
message: `Switching to ${result.newModel?.split("/").pop() || result.newModel} for next request`,
variant: "warning",
duration: 5000,
},
})
.catch(() => {})
}
if (result.success && result.newModel) {
await helpers.autoRetryWithFallback(sessionID, result.newModel, resolvedAgent, "session.status")
}
if (!result.success) {
log(`[${HOOK_NAME}] Fallback preparation failed`, { sessionID, error: result.error })
}
}
return async ({ event }: { event: { type: string; properties?: unknown } }) => {
if (!config.enabled) return
@@ -191,6 +276,7 @@ export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
if (event.type === "session.deleted") { handleSessionDeleted(props); return }
if (event.type === "session.stop") { await handleSessionStop(props); return }
if (event.type === "session.idle") { handleSessionIdle(props); return }
if (event.type === "session.status") { await handleSessionStatus(props); return }
if (event.type === "session.error") { await handleSessionError(props); return }
}
}
@@ -0,0 +1,66 @@
import { afterEach, describe, expect, test } from "bun:test"
import { getFallbackModelsForSession } from "./fallback-models"
import { SessionCategoryRegistry } from "../../shared/session-category-registry"
describe("runtime-fallback fallback-models", () => {
afterEach(() => {
SessionCategoryRegistry.clear()
})
test("uses category fallback_models when session category is registered", () => {
//#given
const sessionID = "ses_runtime_fallback_category"
SessionCategoryRegistry.register(sessionID, "quick")
const pluginConfig = {
categories: {
quick: {
fallback_models: ["openai/gpt-5.2", "anthropic/claude-opus-4-6"],
},
},
} as any
//#when
const result = getFallbackModelsForSession(sessionID, undefined, pluginConfig)
//#then
expect(result).toEqual(["openai/gpt-5.2", "anthropic/claude-opus-4-6"])
})
test("uses agent-specific fallback_models when agent is resolved", () => {
//#given
const pluginConfig = {
agents: {
oracle: {
fallback_models: ["openai/gpt-5.2", "anthropic/claude-opus-4-6"],
},
},
} as any
//#when
const result = getFallbackModelsForSession("ses_runtime_fallback_agent", "oracle", pluginConfig)
//#then
expect(result).toEqual(["openai/gpt-5.2", "anthropic/claude-opus-4-6"])
})
test("does not fall back to another agent chain when agent cannot be resolved", () => {
//#given
const pluginConfig = {
agents: {
sisyphus: {
fallback_models: ["quotio/gpt-5.2", "quotio/glm-5", "quotio/kimi-k2.5"],
},
oracle: {
fallback_models: ["openai/gpt-5.2", "anthropic/claude-opus-4-6"],
},
},
} as any
//#when
const result = getFallbackModelsForSession("ses_runtime_fallback_unknown", undefined, pluginConfig)
//#then
expect(result).toEqual([])
})
})
+2 -14
View File
@@ -1,5 +1,5 @@
import type { OhMyOpenCodeConfig } from "../../config"
import { AGENT_NAMES, agentPattern } from "./agent-resolver"
import { agentPattern } from "./agent-resolver"
import { HOOK_NAME } from "./constants"
import { log } from "../../shared/logger"
import { SessionCategoryRegistry } from "../../shared/session-category-registry"
@@ -51,19 +51,7 @@ export function getFallbackModelsForSession(
if (result) return result
}
const sisyphusFallback = tryGetFallbackFromAgent("sisyphus")
if (sisyphusFallback) {
log(`[${HOOK_NAME}] Using sisyphus fallback models (no agent detected)`, { sessionID })
return sisyphusFallback
}
for (const agentName of AGENT_NAMES) {
const result = tryGetFallbackFromAgent(agentName)
if (result) {
log(`[${HOOK_NAME}] Using ${agentName} fallback models (no agent detected)`, { sessionID })
return result
}
}
log(`[${HOOK_NAME}] No category/agent fallback models resolved for session`, { sessionID, agent })
return []
}
+213
View File
@@ -387,6 +387,219 @@ describe("runtime-fallback", () => {
expect(fallbackLog?.data).toMatchObject({ from: "openai/gpt-5.3-codex", to: "anthropic/claude-opus-4-6" })
})
test("should trigger fallback on auto-retry signal in assistant text parts", async () => {
const hook = createRuntimeFallbackHook(createMockPluginInput(), {
config: createMockConfig({ notify_on_fallback: false }),
pluginConfig: createMockPluginConfigWithCategoryFallback(["openai/gpt-5.2"]),
})
const sessionID = "test-session-parts-auto-retry"
SessionCategoryRegistry.register(sessionID, "test")
await hook.event({
event: {
type: "session.created",
properties: { info: { id: sessionID, model: "quotio/claude-opus-4-6" } },
},
})
await hook.event({
event: {
type: "message.updated",
properties: {
info: {
sessionID,
role: "assistant",
model: "quotio/claude-opus-4-6",
},
parts: [
{
type: "text",
text: "This request would exceed your account's rate limit. Please try again later. [retrying in 2s attempt #2]",
},
],
},
},
})
const signalLog = logCalls.find((c) => c.msg.includes("Detected provider auto-retry signal"))
expect(signalLog).toBeDefined()
const fallbackLog = logCalls.find((c) => c.msg.includes("Preparing fallback"))
expect(fallbackLog).toBeDefined()
expect(fallbackLog?.data).toMatchObject({ from: "quotio/claude-opus-4-6", to: "openai/gpt-5.2" })
})
test("should trigger fallback when auto-retry text parts are nested under info.parts", async () => {
const hook = createRuntimeFallbackHook(createMockPluginInput(), {
config: createMockConfig({ notify_on_fallback: false }),
pluginConfig: createMockPluginConfigWithCategoryFallback(["openai/gpt-5.2"]),
})
const sessionID = "test-session-info-parts-auto-retry"
SessionCategoryRegistry.register(sessionID, "test")
await hook.event({
event: {
type: "session.created",
properties: { info: { id: sessionID, model: "quotio/claude-opus-4-6" } },
},
})
await hook.event({
event: {
type: "message.updated",
properties: {
info: {
sessionID,
role: "assistant",
model: "quotio/claude-opus-4-6",
parts: [
{
type: "text",
text: "This request would exceed your account's rate limit. Please try again later. [retrying in 2s attempt #2]",
},
],
},
},
},
})
const signalLog = logCalls.find((c) => c.msg.includes("Detected provider auto-retry signal"))
expect(signalLog).toBeDefined()
const fallbackLog = logCalls.find((c) => c.msg.includes("Preparing fallback"))
expect(fallbackLog).toBeDefined()
expect(fallbackLog?.data).toMatchObject({ from: "quotio/claude-opus-4-6", to: "openai/gpt-5.2" })
})
test("should trigger fallback on session.status auto-retry signal", async () => {
const promptCalls: unknown[] = []
const hook = createRuntimeFallbackHook(
createMockPluginInput({
session: {
messages: async () => ({
data: [
{
info: { role: "user" },
parts: [{ type: "text", text: "continue" }],
},
],
}),
promptAsync: async (args) => {
promptCalls.push(args)
return {}
},
},
}),
{
config: createMockConfig({ notify_on_fallback: false }),
pluginConfig: createMockPluginConfigWithCategoryFallback(["openai/gpt-5.2"]),
}
)
const sessionID = "test-session-status-auto-retry"
SessionCategoryRegistry.register(sessionID, "test")
await hook.event({
event: {
type: "session.created",
properties: { info: { id: sessionID, model: "quotio/claude-opus-4-6" } },
},
})
await hook.event({
event: {
type: "session.status",
properties: {
sessionID,
status: {
type: "retry",
next: 476,
attempt: 1,
message: "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 56s attempt #1]",
},
},
},
})
const signalLog = logCalls.find((c) => c.msg.includes("Detected provider auto-retry signal in session.status"))
expect(signalLog).toBeDefined()
const fallbackLog = logCalls.find((c) => c.msg.includes("Preparing fallback"))
expect(fallbackLog).toBeDefined()
expect(fallbackLog?.data).toMatchObject({ from: "quotio/claude-opus-4-6", to: "openai/gpt-5.2" })
expect(promptCalls.length).toBe(1)
})
test("should deduplicate session.status countdown updates for the same retry attempt", async () => {
const promptCalls: unknown[] = []
const hook = createRuntimeFallbackHook(
createMockPluginInput({
session: {
messages: async () => ({
data: [
{
info: { role: "user" },
parts: [{ type: "text", text: "continue" }],
},
],
}),
promptAsync: async (args) => {
promptCalls.push(args)
return {}
},
},
}),
{
config: createMockConfig({ notify_on_fallback: false }),
pluginConfig: createMockPluginConfigWithCategoryFallback(["openai/gpt-5.2"]),
}
)
const sessionID = "test-session-status-dedup"
SessionCategoryRegistry.register(sessionID, "test")
await hook.event({
event: {
type: "session.created",
properties: { info: { id: sessionID, model: "quotio/claude-opus-4-6" } },
},
})
await hook.event({
event: {
type: "session.status",
properties: {
sessionID,
status: {
type: "retry",
next: 476,
attempt: 1,
message: "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 56s attempt #1]",
},
},
},
})
await hook.event({
event: {
type: "session.status",
properties: {
sessionID,
status: {
type: "retry",
next: 475,
attempt: 1,
message: "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 55s attempt #1]",
},
},
},
})
expect(promptCalls.length).toBe(1)
})
test("should NOT trigger fallback on auto-retry signal when timeout_seconds is 0", async () => {
const hook = createRuntimeFallbackHook(createMockPluginInput(), {
config: createMockConfig({ notify_on_fallback: false, timeout_seconds: 0 }),
@@ -57,10 +57,20 @@ export function createMessageUpdateHandler(deps: HookDeps, helpers: AutoRetryHel
return async (props: Record<string, unknown> | undefined) => {
const info = props?.info as Record<string, unknown> | undefined
const sessionID = info?.sessionID as string | undefined
const retrySignalResult = extractAutoRetrySignal(info)
const retrySignal = retrySignalResult?.signal
const timeoutEnabled = config.timeout_seconds > 0
const parts = props?.parts as Array<{ type?: string; text?: string }> | undefined
const eventParts = props?.parts as Array<{ type?: string; text?: string }> | undefined
const infoParts = info?.parts as Array<{ type?: string; text?: string }> | undefined
const parts = eventParts && eventParts.length > 0 ? eventParts : infoParts
const retrySignalResult = extractAutoRetrySignal(info)
const partsText = (parts ?? [])
.filter((p) => typeof p?.text === "string")
.map((p) => (p.text ?? "").trim())
.filter((text) => text.length > 0)
.join("\n")
const retrySignalFromParts = partsText
? extractAutoRetrySignal({ message: partsText, status: partsText, summary: partsText })?.signal
: undefined
const retrySignal = retrySignalResult?.signal ?? retrySignalFromParts
const errorContentResult = containsErrorContent(parts)
const error = info?.error ??
(retrySignal && timeoutEnabled ? { name: "ProviderRateLimitError", message: retrySignal } : undefined) ??