fix(runtime-fallback): advance session.status fallback chain
Allow provider cooldown events to override a pending fallback retry so runtime fallback can keep progressing instead of stalling on the same model. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
@@ -0,0 +1,111 @@
|
|||||||
|
import { describe, expect, it } from "bun:test"
|
||||||
|
import type { HookDeps, RuntimeFallbackPluginInput } from "./types"
|
||||||
|
import type { AutoRetryHelpers } from "./auto-retry"
|
||||||
|
import { createFallbackState } from "./fallback-state"
|
||||||
|
import { createSessionStatusHandler } from "./session-status-handler"
|
||||||
|
import { SessionCategoryRegistry } from "../../shared/session-category-registry"
|
||||||
|
|
||||||
|
function createContext(): RuntimeFallbackPluginInput {
|
||||||
|
return {
|
||||||
|
client: {
|
||||||
|
session: {
|
||||||
|
abort: async () => ({}),
|
||||||
|
messages: async () => ({ data: [] }),
|
||||||
|
promptAsync: async () => ({}),
|
||||||
|
},
|
||||||
|
tui: {
|
||||||
|
showToast: async () => ({}),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
directory: "/test/dir",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function createDeps(): HookDeps {
|
||||||
|
return {
|
||||||
|
ctx: createContext(),
|
||||||
|
config: {
|
||||||
|
enabled: true,
|
||||||
|
retry_on_errors: [429, 503, 529],
|
||||||
|
max_fallback_attempts: 4,
|
||||||
|
cooldown_seconds: 60,
|
||||||
|
timeout_seconds: 30,
|
||||||
|
notify_on_fallback: false,
|
||||||
|
},
|
||||||
|
options: undefined,
|
||||||
|
pluginConfig: {
|
||||||
|
categories: {
|
||||||
|
test: {
|
||||||
|
fallback_models: ["openai/gpt-5.4", "google/gemini-2.5-pro"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
sessionStates: new Map(),
|
||||||
|
sessionLastAccess: new Map(),
|
||||||
|
sessionRetryInFlight: new Set(),
|
||||||
|
sessionAwaitingFallbackResult: new Set(),
|
||||||
|
sessionFallbackTimeouts: new Map(),
|
||||||
|
sessionStatusRetryKeys: new Map(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function createHelpers(abortCalls: string[], retryCalls: Array<{ sessionID: string; model: string; source: string }>): AutoRetryHelpers {
|
||||||
|
return {
|
||||||
|
abortSessionRequest: async (sessionID: string) => {
|
||||||
|
abortCalls.push(sessionID)
|
||||||
|
},
|
||||||
|
clearSessionFallbackTimeout: () => {},
|
||||||
|
scheduleSessionFallbackTimeout: () => {},
|
||||||
|
autoRetryWithFallback: async (sessionID: string, model: string, _resolvedAgent: string | undefined, source: string) => {
|
||||||
|
retryCalls.push({ sessionID, model, source })
|
||||||
|
},
|
||||||
|
resolveAgentForSessionFromContext: async () => undefined,
|
||||||
|
cleanupStaleSessions: () => {},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("createSessionStatusHandler", () => {
|
||||||
|
it("#given a pending fallback model #when a new provider cooldown retry arrives #then the handler overrides the pending fallback and advances the chain", async () => {
|
||||||
|
// given
|
||||||
|
SessionCategoryRegistry.clear()
|
||||||
|
const sessionID = "session-status-pending-fallback"
|
||||||
|
SessionCategoryRegistry.register(sessionID, "test")
|
||||||
|
|
||||||
|
const deps = createDeps()
|
||||||
|
const abortCalls: string[] = []
|
||||||
|
const retryCalls: Array<{ sessionID: string; model: string; source: string }> = []
|
||||||
|
const state = createFallbackState("anthropic/claude-opus-4-6")
|
||||||
|
state.currentModel = "openai/gpt-5.4"
|
||||||
|
state.fallbackIndex = 0
|
||||||
|
state.attemptCount = 1
|
||||||
|
state.pendingFallbackModel = "openai/gpt-5.4"
|
||||||
|
state.failedModels.set("anthropic/claude-opus-4-6", Date.now())
|
||||||
|
deps.sessionStates.set(sessionID, state)
|
||||||
|
|
||||||
|
const handler = createSessionStatusHandler(deps, createHelpers(abortCalls, retryCalls), deps.sessionStatusRetryKeys)
|
||||||
|
|
||||||
|
// when
|
||||||
|
await handler({
|
||||||
|
sessionID,
|
||||||
|
model: "openai/gpt-5.4",
|
||||||
|
status: {
|
||||||
|
type: "retry",
|
||||||
|
attempt: 2,
|
||||||
|
message: "All credentials for model gpt-5.4 are cooling down [retrying in 7m 56s attempt #2]",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
// then
|
||||||
|
expect(abortCalls).toEqual([sessionID])
|
||||||
|
expect(retryCalls).toEqual([
|
||||||
|
{
|
||||||
|
sessionID,
|
||||||
|
model: "google/gemini-2.5-pro",
|
||||||
|
source: "session.status",
|
||||||
|
},
|
||||||
|
])
|
||||||
|
expect(state.currentModel).toBe("google/gemini-2.5-pro")
|
||||||
|
expect(state.pendingFallbackModel).toBe("google/gemini-2.5-pro")
|
||||||
|
SessionCategoryRegistry.clear()
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -26,6 +26,7 @@ export function createSessionStatusHandler(
|
|||||||
const status = props?.status as { type?: string; message?: string; attempt?: number } | undefined
|
const status = props?.status as { type?: string; message?: string; attempt?: number } | undefined
|
||||||
const agent = props?.agent as string | undefined
|
const agent = props?.agent as string | undefined
|
||||||
const model = props?.model as string | undefined
|
const model = props?.model as string | undefined
|
||||||
|
const timeoutEnabled = deps.config.timeout_seconds > 0
|
||||||
|
|
||||||
if (!sessionID || status?.type !== "retry") return
|
if (!sessionID || status?.type !== "retry") return
|
||||||
|
|
||||||
@@ -40,8 +41,17 @@ export function createSessionStatusHandler(
|
|||||||
sessionStatusRetryKeys.set(sessionID, retryKey)
|
sessionStatusRetryKeys.set(sessionID, retryKey)
|
||||||
|
|
||||||
if (sessionRetryInFlight.has(sessionID)) {
|
if (sessionRetryInFlight.has(sessionID)) {
|
||||||
log(`[${HOOK_NAME}] session.status retry skipped — retry already in flight`, { sessionID })
|
if (timeoutEnabled) {
|
||||||
return
|
log(`[${HOOK_NAME}] Overriding in-flight retry due to provider auto-retry signal`, {
|
||||||
|
sessionID,
|
||||||
|
model,
|
||||||
|
})
|
||||||
|
await helpers.abortSessionRequest(sessionID, "session.status.retry-signal")
|
||||||
|
sessionRetryInFlight.delete(sessionID)
|
||||||
|
} else {
|
||||||
|
log(`[${HOOK_NAME}] session.status retry skipped — retry already in flight`, { sessionID })
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const resolvedAgent = await helpers.resolveAgentForSessionFromContext(sessionID, agent)
|
const resolvedAgent = await helpers.resolveAgentForSessionFromContext(sessionID, agent)
|
||||||
@@ -75,11 +85,19 @@ export function createSessionStatusHandler(
|
|||||||
sessionLastAccess.set(sessionID, Date.now())
|
sessionLastAccess.set(sessionID, Date.now())
|
||||||
|
|
||||||
if (state.pendingFallbackModel) {
|
if (state.pendingFallbackModel) {
|
||||||
log(`[${HOOK_NAME}] session.status retry skipped (pending fallback in progress)`, {
|
if (timeoutEnabled) {
|
||||||
sessionID,
|
log(`[${HOOK_NAME}] Clearing pending fallback due to provider auto-retry signal`, {
|
||||||
pendingFallbackModel: state.pendingFallbackModel,
|
sessionID,
|
||||||
})
|
pendingFallbackModel: state.pendingFallbackModel,
|
||||||
return
|
})
|
||||||
|
state.pendingFallbackModel = undefined
|
||||||
|
} else {
|
||||||
|
log(`[${HOOK_NAME}] session.status retry skipped (pending fallback in progress)`, {
|
||||||
|
sessionID,
|
||||||
|
pendingFallbackModel: state.pendingFallbackModel,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
log(`[${HOOK_NAME}] Detected provider auto-retry signal in session.status`, {
|
log(`[${HOOK_NAME}] Detected provider auto-retry signal in session.status`, {
|
||||||
|
|||||||
Reference in New Issue
Block a user