Fix cooldown fallback switching across model/runtime fallback hooks

2026-03-04 18:35:09 +01:00
parent eb43bd7c43
commit 512bc05404
11 changed files with 803 additions and 38 deletions
@@ -26,6 +26,10 @@ export const RETRYABLE_ERROR_PATTERNS = [
  /rate.?limit/i,
  /too.?many.?requests/i,
  /quota.?exceeded/i,
+  /quota\s+will\s+reset\s+after/i,
+  /all\s+credentials\s+for\s+model/i,
+  /cool(?:ing)?\s+down/i,
+  /exhausted\s+your\s+capacity/i,
  /usage\s+limit\s+has\s+been\s+reached/i,
  /service.?unavailable/i,
  /overloaded/i,
@@ -0,0 +1,46 @@
+import { describe, expect, test } from "bun:test"
+
+import { extractAutoRetrySignal, isRetryableError } from "./error-classifier"
+
+describe("runtime-fallback error classifier", () => {
+  test("detects cooling-down auto-retry status signals", () => {
+    //#given
+    const info = {
+      status:
+        "All credentials for model claude-opus-4-6-thinking are cooling down [retrying in ~5 days attempt #1]",
+    }
+
+    //#when
+    const signal = extractAutoRetrySignal(info)
+
+    //#then
+    expect(signal).toBeDefined()
+  })
+
+  test("treats cooling-down retry messages as retryable", () => {
+    //#given
+    const error = {
+      message:
+        "All credentials for model claude-opus-4-6-thinking are cooling down [retrying in ~5 days attempt #1]",
+    }
+
+    //#when
+    const retryable = isRetryableError(error, [400, 403, 408, 429, 500, 502, 503, 504, 529])
+
+    //#then
+    expect(retryable).toBe(true)
+  })
+
+  test("ignores non-retry assistant status text", () => {
+    //#given
+    const info = {
+      status: "Thinking...",
+    }
+
+    //#when
+    const signal = extractAutoRetrySignal(info)
+
+    //#then
+    expect(signal).toBeUndefined()
+  })
+})
@@ -102,7 +102,7 @@ export interface AutoRetrySignal {
 export const AUTO_RETRY_PATTERNS: Array<(combined: string) => boolean> = [
  (combined) => /retrying\s+in/i.test(combined),
  (combined) =>
-    /(?:too\s+many\s+requests|quota\s*exceeded|usage\s+limit|rate\s+limit|limit\s+reached)/i.test(combined),
+    /(?:too\s+many\s+requests|quota\s*exceeded|quota\s+will\s+reset\s+after|usage\s+limit|rate\s+limit|limit\s+reached|all\s+credentials\s+for\s+model|cool(?:ing)?\s+down|exhausted\s+your\s+capacity)/i.test(combined),
 ]

 export function extractAutoRetrySignal(info: Record<string, unknown> | undefined): AutoRetrySignal | undefined {
@@ -2,13 +2,14 @@ import type { HookDeps } from "./types"
 import type { AutoRetryHelpers } from "./auto-retry"
 import { HOOK_NAME } from "./constants"
 import { log } from "../../shared/logger"
-import { extractStatusCode, extractErrorName, classifyErrorType, isRetryableError } from "./error-classifier"
+import { extractStatusCode, extractErrorName, classifyErrorType, isRetryableError, extractAutoRetrySignal } from "./error-classifier"
 import { createFallbackState, prepareFallback } from "./fallback-state"
 import { getFallbackModelsForSession } from "./fallback-models"
 import { SessionCategoryRegistry } from "../../shared/session-category-registry"

 export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
  const { config, pluginConfig, sessionStates, sessionLastAccess, sessionRetryInFlight, sessionAwaitingFallbackResult, sessionFallbackTimeouts } = deps
+  const sessionStatusRetryKeys = new Map<string, string>()

  const handleSessionCreated = (props: Record<string, unknown> | undefined) => {
    const sessionInfo = props?.info as { id?: string; model?: string } | undefined
@@ -33,6 +34,7 @@ export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
      sessionRetryInFlight.delete(sessionID)
      sessionAwaitingFallbackResult.delete(sessionID)
      helpers.clearSessionFallbackTimeout(sessionID)
+      sessionStatusRetryKeys.delete(sessionID)
      SessionCategoryRegistry.remove(sessionID)
    }
  }
@@ -182,6 +184,104 @@ export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
    }
  }

+  const normalizeRetryStatusMessage = (message: string): string =>
+    message
+      .replace(/\[retrying in [^\]]*attempt\s*#\d+\]/gi, "[retrying]")
+      .replace(/retrying in\s+[^(]*attempt\s*#\d+/gi, "retrying")
+      .replace(/\s+/g, " ")
+      .trim()
+      .toLowerCase()
+
+  const extractRetryAttempt = (statusAttempt: unknown, message: string): string => {
+    if (typeof statusAttempt === "number" && Number.isFinite(statusAttempt)) {
+      return String(statusAttempt)
+    }
+    const match = message.match(/attempt\s*#\s*(\d+)/i)
+    return match?.[1] ?? "?"
+  }
+
+  const handleSessionStatus = async (props: Record<string, unknown> | undefined) => {
+    const sessionID = props?.sessionID as string | undefined
+    const status = props?.status as { type?: string; message?: string; attempt?: number } | undefined
+    const agent = props?.agent as string | undefined
+    const model = props?.model as string | undefined
+
+    if (!sessionID || status?.type !== "retry") return
+
+    const retryMessage = typeof status.message === "string" ? status.message : ""
+    const retrySignal = extractAutoRetrySignal({ status: retryMessage, message: retryMessage })
+    if (!retrySignal) return
+
+    const retryKey = `${extractRetryAttempt(status.attempt, retryMessage)}:${normalizeRetryStatusMessage(retryMessage)}`
+    if (sessionStatusRetryKeys.get(sessionID) === retryKey) {
+      return
+    }
+    sessionStatusRetryKeys.set(sessionID, retryKey)
+
+    if (sessionRetryInFlight.has(sessionID)) {
+      log(`[${HOOK_NAME}] session.status retry skipped — retry already in flight`, { sessionID })
+      return
+    }
+
+    const resolvedAgent = await helpers.resolveAgentForSessionFromContext(sessionID, agent)
+    const fallbackModels = getFallbackModelsForSession(sessionID, resolvedAgent, pluginConfig)
+    if (fallbackModels.length === 0) return
+
+    let state = sessionStates.get(sessionID)
+    if (!state) {
+      const detectedAgent = resolvedAgent
+      const agentConfig = detectedAgent
+        ? pluginConfig?.agents?.[detectedAgent as keyof typeof pluginConfig.agents]
+        : undefined
+      const inferredModel = model || (agentConfig?.model as string | undefined)
+      if (!inferredModel) {
+        log(`[${HOOK_NAME}] session.status retry missing model info, cannot fallback`, { sessionID })
+        return
+      }
+      state = createFallbackState(inferredModel)
+      sessionStates.set(sessionID, state)
+    }
+    sessionLastAccess.set(sessionID, Date.now())
+
+    if (state.pendingFallbackModel) {
+      log(`[${HOOK_NAME}] session.status retry skipped (pending fallback in progress)`, {
+        sessionID,
+        pendingFallbackModel: state.pendingFallbackModel,
+      })
+      return
+    }
+
+    log(`[${HOOK_NAME}] Detected provider auto-retry signal in session.status`, {
+      sessionID,
+      model: state.currentModel,
+      retryAttempt: status.attempt,
+    })
+
+    await helpers.abortSessionRequest(sessionID, "session.status.retry-signal")
+
+    const result = prepareFallback(sessionID, state, fallbackModels, config)
+    if (result.success && config.notify_on_fallback) {
+      await deps.ctx.client.tui
+        .showToast({
+          body: {
+            title: "Model Fallback",
+            message: `Switching to ${result.newModel?.split("/").pop() || result.newModel} for next request`,
+            variant: "warning",
+            duration: 5000,
+          },
+        })
+        .catch(() => {})
+    }
+
+    if (result.success && result.newModel) {
+      await helpers.autoRetryWithFallback(sessionID, result.newModel, resolvedAgent, "session.status")
+    }
+
+    if (!result.success) {
+      log(`[${HOOK_NAME}] Fallback preparation failed`, { sessionID, error: result.error })
+    }
+  }
+
  return async ({ event }: { event: { type: string; properties?: unknown } }) => {
    if (!config.enabled) return

@@ -191,6 +291,7 @@ export function createEventHandler(deps: HookDeps, helpers: AutoRetryHelpers) {
    if (event.type === "session.deleted") { handleSessionDeleted(props); return }
    if (event.type === "session.stop") { await handleSessionStop(props); return }
    if (event.type === "session.idle") { handleSessionIdle(props); return }
+    if (event.type === "session.status") { await handleSessionStatus(props); return }
    if (event.type === "session.error") { await handleSessionError(props); return }
  }
 }
@@ -387,6 +387,130 @@ describe("runtime-fallback", () => {
      expect(fallbackLog?.data).toMatchObject({ from: "openai/gpt-5.3-codex", to: "anthropic/claude-opus-4-6" })
    })

+    test("should trigger fallback on session.status auto-retry signal", async () => {
+      const promptCalls: unknown[] = []
+      const hook = createRuntimeFallbackHook(
+        createMockPluginInput({
+          session: {
+            messages: async () => ({
+              data: [
+                {
+                  info: { role: "user" },
+                  parts: [{ type: "text", text: "continue" }],
+                },
+              ],
+            }),
+            promptAsync: async (args) => {
+              promptCalls.push(args)
+              return {}
+            },
+          },
+        }),
+        {
+          config: createMockConfig({ notify_on_fallback: false }),
+          pluginConfig: createMockPluginConfigWithCategoryFallback(["openai/gpt-5.2"]),
+        }
+      )
+
+      const sessionID = "test-session-status-auto-retry"
+      SessionCategoryRegistry.register(sessionID, "test")
+
+      await hook.event({
+        event: {
+          type: "session.created",
+          properties: { info: { id: sessionID, model: "quotio/claude-opus-4-6" } },
+        },
+      })
+
+      await hook.event({
+        event: {
+          type: "session.status",
+          properties: {
+            sessionID,
+            status: {
+              type: "retry",
+              attempt: 1,
+              message: "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 56s attempt #1]",
+            },
+          },
+        },
+      })
+
+      const signalLog = logCalls.find((c) => c.msg.includes("Detected provider auto-retry signal in session.status"))
+      expect(signalLog).toBeDefined()
+
+      const fallbackLog = logCalls.find((c) => c.msg.includes("Preparing fallback"))
+      expect(fallbackLog).toBeDefined()
+      expect(fallbackLog?.data).toMatchObject({ from: "quotio/claude-opus-4-6", to: "openai/gpt-5.2" })
+      expect(promptCalls.length).toBe(1)
+    })
+
+    test("should deduplicate session.status countdown updates for the same retry attempt", async () => {
+      const promptCalls: unknown[] = []
+      const hook = createRuntimeFallbackHook(
+        createMockPluginInput({
+          session: {
+            messages: async () => ({
+              data: [
+                {
+                  info: { role: "user" },
+                  parts: [{ type: "text", text: "continue" }],
+                },
+              ],
+            }),
+            promptAsync: async (args) => {
+              promptCalls.push(args)
+              return {}
+            },
+          },
+        }),
+        {
+          config: createMockConfig({ notify_on_fallback: false }),
+          pluginConfig: createMockPluginConfigWithCategoryFallback(["openai/gpt-5.2"]),
+        }
+      )
+
+      const sessionID = "test-session-status-dedup"
+      SessionCategoryRegistry.register(sessionID, "test")
+
+      await hook.event({
+        event: {
+          type: "session.created",
+          properties: { info: { id: sessionID, model: "quotio/claude-opus-4-6" } },
+        },
+      })
+
+      await hook.event({
+        event: {
+          type: "session.status",
+          properties: {
+            sessionID,
+            status: {
+              type: "retry",
+              attempt: 1,
+              message: "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 56s attempt #1]",
+            },
+          },
+        },
+      })
+
+      await hook.event({
+        event: {
+          type: "session.status",
+          properties: {
+            sessionID,
+            status: {
+              type: "retry",
+              attempt: 1,
+              message: "All credentials for model claude-opus-4-6 are cooling down [retrying in 7m 55s attempt #1]",
+            },
+          },
+        },
+      })
+
+      expect(promptCalls.length).toBe(1)
+    })
+
    test("should NOT trigger fallback on auto-retry signal when timeout_seconds is 0", async () => {
      const hook = createRuntimeFallbackHook(createMockPluginInput(), {
        config: createMockConfig({ notify_on_fallback: false, timeout_seconds: 0 }),