fix(fallback): preserve provider-specific fallback retries
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
@@ -301,6 +301,68 @@ describe("createEventHandler - model fallback", () => {
|
|||||||
expect(abortCalls).toEqual([sessionID])
|
expect(abortCalls).toEqual([sessionID])
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test("does not collapse fallback continuations for different providers with the same model id", async () => {
|
||||||
|
//#given
|
||||||
|
const sessionID = "ses_model_fallback_same_model_different_provider"
|
||||||
|
setMainSession(sessionID)
|
||||||
|
let pendingFallbackArms = 0
|
||||||
|
const modelFallback = unsafeTestValue({
|
||||||
|
setSessionFallbackChain: () => {},
|
||||||
|
setPendingModelFallback: () => {
|
||||||
|
pendingFallbackArms += 1
|
||||||
|
return true
|
||||||
|
},
|
||||||
|
})
|
||||||
|
const { handler, abortCalls, promptAsyncCalls } = createHandler({
|
||||||
|
hooks: { modelFallback },
|
||||||
|
promptAsync: async () => ({}),
|
||||||
|
})
|
||||||
|
|
||||||
|
const assistantError = {
|
||||||
|
name: "APIError",
|
||||||
|
data: {
|
||||||
|
message:
|
||||||
|
"Bad Gateway: {\"error\":{\"message\":\"unknown provider for model claude-opus-4-7-thinking\"}}",
|
||||||
|
isRetryable: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
await handler({
|
||||||
|
event: {
|
||||||
|
type: "message.updated",
|
||||||
|
properties: {
|
||||||
|
info: {
|
||||||
|
id: "msg_err_same_model_provider_1",
|
||||||
|
sessionID,
|
||||||
|
role: "assistant",
|
||||||
|
error: assistantError,
|
||||||
|
modelID: "claude-opus-4-7-thinking",
|
||||||
|
providerID: "anthropic",
|
||||||
|
agent: "Sisyphus - Ultraworker",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
//#when - a distinct provider reports the same normalized model id before idle cleanup
|
||||||
|
await handler({
|
||||||
|
event: {
|
||||||
|
type: "session.error",
|
||||||
|
properties: {
|
||||||
|
sessionID,
|
||||||
|
providerID: "quotio",
|
||||||
|
modelID: "claude-opus-4-7-thinking",
|
||||||
|
error: assistantError,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
//#then
|
||||||
|
expect(pendingFallbackArms).toBe(2)
|
||||||
|
expect(promptAsyncCalls).toEqual([sessionID, sessionID])
|
||||||
|
expect(abortCalls).toEqual([sessionID, sessionID])
|
||||||
|
})
|
||||||
|
|
||||||
test("triggers retry prompt on session.status retry events and applies fallback", async () => {
|
test("triggers retry prompt on session.status retry events and applies fallback", async () => {
|
||||||
//#given
|
//#given
|
||||||
const sessionID = "ses_status_retry_fallback"
|
const sessionID = "ses_status_retry_fallback"
|
||||||
|
|||||||
+65
-22
@@ -54,6 +54,17 @@ type FirstMessageVariantGate = {
|
|||||||
clear: (sessionID: string) => void;
|
clear: (sessionID: string) => void;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type FallbackContinuationDedupeKeys = {
|
||||||
|
modelKey?: string;
|
||||||
|
providerModelKey?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type FallbackContinuationDedupeState = {
|
||||||
|
modelKeys: Set<string>;
|
||||||
|
providerModelKeys: Set<string>;
|
||||||
|
providerlessModelKeys: Set<string>;
|
||||||
|
};
|
||||||
|
|
||||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
return typeof value === "object" && value !== null;
|
return typeof value === "object" && value !== null;
|
||||||
}
|
}
|
||||||
@@ -220,7 +231,7 @@ export function createEventHandler(args: {
|
|||||||
const lastHandledRetryStatusKey = new Map<string, string>();
|
const lastHandledRetryStatusKey = new Map<string, string>();
|
||||||
const lastKnownModelBySession = new Map<string, { providerID: string; modelID: string }>();
|
const lastKnownModelBySession = new Map<string, { providerID: string; modelID: string }>();
|
||||||
const modelFallbackContinuationsInFlight = new Set<string>();
|
const modelFallbackContinuationsInFlight = new Set<string>();
|
||||||
const lastDispatchedModelFallbackContinuationKeys = new Map<string, Set<string>>();
|
const lastDispatchedModelFallbackContinuationKeys = new Map<string, FallbackContinuationDedupeState>();
|
||||||
|
|
||||||
const resolveFallbackProviderID = (sessionID: string, providerHint?: string): string => {
|
const resolveFallbackProviderID = (sessionID: string, providerHint?: string): string => {
|
||||||
const normalizedProviderHint = providerHint?.trim();
|
const normalizedProviderHint = providerHint?.trim();
|
||||||
@@ -364,23 +375,53 @@ export function createEventHandler(args: {
|
|||||||
const getFallbackContinuationKeys = (fallbackContext?: {
|
const getFallbackContinuationKeys = (fallbackContext?: {
|
||||||
agentName?: string;
|
agentName?: string;
|
||||||
providerID?: string;
|
providerID?: string;
|
||||||
|
dedupeProviderID?: string;
|
||||||
modelID?: string;
|
modelID?: string;
|
||||||
}): string[] => {
|
}): FallbackContinuationDedupeKeys => {
|
||||||
const agentKey = fallbackContext?.agentName
|
const agentKey = fallbackContext?.agentName
|
||||||
? getAgentConfigKey(fallbackContext.agentName).trim().toLowerCase()
|
? getAgentConfigKey(fallbackContext.agentName).trim().toLowerCase()
|
||||||
: "";
|
: "";
|
||||||
const providerID = fallbackContext?.providerID?.trim().toLowerCase() ?? "";
|
const providerID = fallbackContext?.dedupeProviderID?.trim().toLowerCase() ?? "";
|
||||||
const modelID = fallbackContext?.modelID?.trim().toLowerCase() ?? "";
|
const modelID = fallbackContext?.modelID?.trim().toLowerCase() ?? "";
|
||||||
|
|
||||||
if (!agentKey || !modelID) {
|
if (!agentKey || !modelID) {
|
||||||
return [];
|
return {};
|
||||||
}
|
}
|
||||||
|
|
||||||
const keys = [`${agentKey}:${modelID}`];
|
return {
|
||||||
if (providerID) {
|
modelKey: `${agentKey}:${modelID}`,
|
||||||
keys.push(`${agentKey}:${providerID}:${modelID}`);
|
...(providerID ? { providerModelKey: `${agentKey}:${providerID}:${modelID}` } : {}),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const getFallbackContinuationDedupeState = (sessionID: string): FallbackContinuationDedupeState => {
|
||||||
|
const existingState = lastDispatchedModelFallbackContinuationKeys.get(sessionID);
|
||||||
|
if (existingState) {
|
||||||
|
return existingState;
|
||||||
}
|
}
|
||||||
return keys;
|
|
||||||
|
const state = {
|
||||||
|
modelKeys: new Set<string>(),
|
||||||
|
providerModelKeys: new Set<string>(),
|
||||||
|
providerlessModelKeys: new Set<string>(),
|
||||||
|
};
|
||||||
|
lastDispatchedModelFallbackContinuationKeys.set(sessionID, state);
|
||||||
|
return state;
|
||||||
|
};
|
||||||
|
|
||||||
|
const wasFallbackContinuationAlreadyDispatched = (
|
||||||
|
state: FallbackContinuationDedupeState | undefined,
|
||||||
|
keys: FallbackContinuationDedupeKeys,
|
||||||
|
): boolean => {
|
||||||
|
if (!state || !keys.modelKey) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!keys.providerModelKey) {
|
||||||
|
return state.modelKeys.has(keys.modelKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
return state.providerModelKeys.has(keys.providerModelKey) || state.providerlessModelKeys.has(keys.modelKey);
|
||||||
};
|
};
|
||||||
|
|
||||||
const autoContinueAfterFallback = async (
|
const autoContinueAfterFallback = async (
|
||||||
@@ -389,6 +430,7 @@ export function createEventHandler(args: {
|
|||||||
fallbackContext?: {
|
fallbackContext?: {
|
||||||
agentName?: string;
|
agentName?: string;
|
||||||
providerID?: string;
|
providerID?: string;
|
||||||
|
dedupeProviderID?: string;
|
||||||
modelID?: string;
|
modelID?: string;
|
||||||
},
|
},
|
||||||
): Promise<void> => {
|
): Promise<void> => {
|
||||||
@@ -400,7 +442,7 @@ export function createEventHandler(args: {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const lastDispatchedKeys = lastDispatchedModelFallbackContinuationKeys.get(sessionID);
|
const lastDispatchedKeys = lastDispatchedModelFallbackContinuationKeys.get(sessionID);
|
||||||
if (lastDispatchedKeys && fallbackKeys.some((fallbackKey) => lastDispatchedKeys.has(fallbackKey))) {
|
if (wasFallbackContinuationAlreadyDispatched(lastDispatchedKeys, fallbackKeys)) {
|
||||||
log("[event] model-fallback continuation skipped because matching fallback was already dispatched", {
|
log("[event] model-fallback continuation skipped because matching fallback was already dispatched", {
|
||||||
sessionID,
|
sessionID,
|
||||||
source,
|
source,
|
||||||
@@ -456,12 +498,14 @@ export function createEventHandler(args: {
|
|||||||
log("[event] model-fallback prompt failed", { sessionID, source, error });
|
log("[event] model-fallback prompt failed", { sessionID, source, error });
|
||||||
});
|
});
|
||||||
} finally {
|
} finally {
|
||||||
if (dispatched && fallbackKeys.length > 0) {
|
if (dispatched && fallbackKeys.modelKey) {
|
||||||
const dispatchedKeys = lastDispatchedModelFallbackContinuationKeys.get(sessionID) ?? new Set<string>();
|
const dispatchedKeys = getFallbackContinuationDedupeState(sessionID);
|
||||||
for (const fallbackKey of fallbackKeys) {
|
dispatchedKeys.modelKeys.add(fallbackKeys.modelKey);
|
||||||
dispatchedKeys.add(fallbackKey);
|
if (fallbackKeys.providerModelKey) {
|
||||||
|
dispatchedKeys.providerModelKeys.add(fallbackKeys.providerModelKey);
|
||||||
|
} else {
|
||||||
|
dispatchedKeys.providerlessModelKeys.add(fallbackKeys.modelKey);
|
||||||
}
|
}
|
||||||
lastDispatchedModelFallbackContinuationKeys.set(sessionID, dispatchedKeys);
|
|
||||||
}
|
}
|
||||||
modelFallbackContinuationsInFlight.delete(sessionID);
|
modelFallbackContinuationsInFlight.delete(sessionID);
|
||||||
}
|
}
|
||||||
@@ -702,10 +746,8 @@ export function createEventHandler(args: {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (agentName) {
|
if (agentName) {
|
||||||
const currentProvider = resolveFallbackProviderID(
|
const providerHint = info?.providerID as string | undefined;
|
||||||
sessionID,
|
const currentProvider = resolveFallbackProviderID(sessionID, providerHint);
|
||||||
info?.providerID as string | undefined,
|
|
||||||
);
|
|
||||||
const rawModel = (info?.modelID as string | undefined) ?? "claude-opus-4-7";
|
const rawModel = (info?.modelID as string | undefined) ?? "claude-opus-4-7";
|
||||||
const currentModel = normalizeFallbackModelID(rawModel);
|
const currentModel = normalizeFallbackModelID(rawModel);
|
||||||
applyUserConfiguredFallbackChain(modelFallback, sessionID, agentName, currentProvider, args.pluginConfig);
|
applyUserConfiguredFallbackChain(modelFallback, sessionID, agentName, currentProvider, args.pluginConfig);
|
||||||
@@ -723,6 +765,7 @@ export function createEventHandler(args: {
|
|||||||
await autoContinueAfterFallback(sessionID, "message.updated", {
|
await autoContinueAfterFallback(sessionID, "message.updated", {
|
||||||
agentName,
|
agentName,
|
||||||
providerID: currentProvider,
|
providerID: currentProvider,
|
||||||
|
dedupeProviderID: providerHint,
|
||||||
modelID: currentModel,
|
modelID: currentModel,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -792,6 +835,7 @@ export function createEventHandler(args: {
|
|||||||
await autoContinueAfterFallback(sessionID, "session.status", {
|
await autoContinueAfterFallback(sessionID, "session.status", {
|
||||||
agentName,
|
agentName,
|
||||||
providerID: currentProvider,
|
providerID: currentProvider,
|
||||||
|
dedupeProviderID: parsed.providerID,
|
||||||
modelID: currentModel,
|
modelID: currentModel,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -864,10 +908,8 @@ export function createEventHandler(args: {
|
|||||||
|
|
||||||
if (agentName) {
|
if (agentName) {
|
||||||
const parsed = extractProviderModelFromErrorMessage(errorMessage);
|
const parsed = extractProviderModelFromErrorMessage(errorMessage);
|
||||||
const currentProvider = resolveFallbackProviderID(
|
const providerHint = (props?.providerID as string | undefined) || parsed.providerID;
|
||||||
sessionID,
|
const currentProvider = resolveFallbackProviderID(sessionID, providerHint);
|
||||||
(props?.providerID as string | undefined) || parsed.providerID,
|
|
||||||
);
|
|
||||||
let currentModel = (props?.modelID as string) || parsed.modelID || "claude-opus-4-7";
|
let currentModel = (props?.modelID as string) || parsed.modelID || "claude-opus-4-7";
|
||||||
currentModel = normalizeFallbackModelID(currentModel);
|
currentModel = normalizeFallbackModelID(currentModel);
|
||||||
applyUserConfiguredFallbackChain(modelFallback, sessionID, agentName, currentProvider, args.pluginConfig);
|
applyUserConfiguredFallbackChain(modelFallback, sessionID, agentName, currentProvider, args.pluginConfig);
|
||||||
@@ -884,6 +926,7 @@ export function createEventHandler(args: {
|
|||||||
await autoContinueAfterFallback(sessionID, "session.error", {
|
await autoContinueAfterFallback(sessionID, "session.error", {
|
||||||
agentName,
|
agentName,
|
||||||
providerID: currentProvider,
|
providerID: currentProvider,
|
||||||
|
dedupeProviderID: providerHint,
|
||||||
modelID: currentModel,
|
modelID: currentModel,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user