Merge pull request #4030 from code-yeongyu/fix/promptasync-concurrency-20260515
fix(fallback): dedupe overlapping fallback continuations
This commit is contained in:
@@ -1,5 +1,5 @@
|
|||||||
declare const require: (name: string) => any
|
/// <reference types="bun-types" />
|
||||||
const { afterEach, describe, expect, spyOn, test } = require("bun:test")
|
import { afterEach, describe, expect, spyOn, test } from "bun:test"
|
||||||
|
|
||||||
import { createEventHandler } from "./event"
|
import { createEventHandler } from "./event"
|
||||||
import { createChatMessageHandler } from "./chat-message"
|
import { createChatMessageHandler } from "./chat-message"
|
||||||
@@ -8,6 +8,17 @@ import { createModelFallbackHook, clearPendingModelFallback } from "../hooks/mod
|
|||||||
import * as connectedProvidersCache from "../shared/connected-providers-cache"
|
import * as connectedProvidersCache from "../shared/connected-providers-cache"
|
||||||
import { unsafeTestValue } from "../../test-support/unsafe-test-value"
|
import { unsafeTestValue } from "../../test-support/unsafe-test-value"
|
||||||
|
|
||||||
|
type EventInput = { event: { type: string; properties?: unknown } }
|
||||||
|
type EventHandlerInput = Parameters<ReturnType<typeof createEventHandler>>[0]
|
||||||
|
type ChatMessageOutput = {
|
||||||
|
message: Record<string, unknown>
|
||||||
|
parts: Array<{ type: string; text?: string }>
|
||||||
|
}
|
||||||
|
|
||||||
|
function asEventHandlerInput(input: EventInput): EventHandlerInput {
|
||||||
|
return unsafeTestValue<EventHandlerInput>(input)
|
||||||
|
}
|
||||||
|
|
||||||
let readConnectedProvidersCacheSpy: { mockRestore: () => void } | undefined
|
let readConnectedProvidersCacheSpy: { mockRestore: () => void } | undefined
|
||||||
let readProviderModelsCacheSpy: { mockRestore: () => void } | undefined
|
let readProviderModelsCacheSpy: { mockRestore: () => void } | undefined
|
||||||
|
|
||||||
@@ -17,25 +28,40 @@ function setupConnectedProviderCacheMocks(): void {
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("createEventHandler - model fallback", () => {
|
describe("createEventHandler - model fallback", () => {
|
||||||
const createHandler = (args?: { hooks?: any; pluginConfig?: any }) => {
|
const createHandler = (args?: {
|
||||||
|
hooks?: any
|
||||||
|
pluginConfig?: any
|
||||||
|
promptAsync?: (input: { path: { id: string } }) => Promise<unknown>
|
||||||
|
}) => {
|
||||||
setupConnectedProviderCacheMocks()
|
setupConnectedProviderCacheMocks()
|
||||||
const abortCalls: string[] = []
|
const abortCalls: string[] = []
|
||||||
const promptCalls: string[] = []
|
const promptCalls: string[] = []
|
||||||
|
const promptAsyncCalls: string[] = []
|
||||||
|
|
||||||
const handler = createEventHandler({
|
const sessionClient = {
|
||||||
|
abort: async ({ path }: { path: { id: string } }) => {
|
||||||
|
abortCalls.push(path.id)
|
||||||
|
return {}
|
||||||
|
},
|
||||||
|
prompt: async ({ path }: { path: { id: string } }) => {
|
||||||
|
promptCalls.push(path.id)
|
||||||
|
return {}
|
||||||
|
},
|
||||||
|
...(args?.promptAsync
|
||||||
|
? {
|
||||||
|
promptAsync: async (input: { path: { id: string } }) => {
|
||||||
|
promptAsyncCalls.push(input.path.id)
|
||||||
|
return args.promptAsync?.(input)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
|
}
|
||||||
|
|
||||||
|
const eventHandler = createEventHandler({
|
||||||
ctx: unsafeTestValue({
|
ctx: unsafeTestValue({
|
||||||
directory: "/tmp",
|
directory: "/tmp",
|
||||||
client: {
|
client: {
|
||||||
session: {
|
session: sessionClient,
|
||||||
abort: async ({ path }: { path: { id: string } }) => {
|
|
||||||
abortCalls.push(path.id)
|
|
||||||
return {}
|
|
||||||
},
|
|
||||||
prompt: async ({ path }: { path: { id: string } }) => {
|
|
||||||
promptCalls.push(path.id)
|
|
||||||
return {}
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
pluginConfig: unsafeTestValue((args?.pluginConfig ?? {})),
|
pluginConfig: unsafeTestValue((args?.pluginConfig ?? {})),
|
||||||
@@ -54,8 +80,9 @@ describe("createEventHandler - model fallback", () => {
|
|||||||
}),
|
}),
|
||||||
hooks: args?.hooks ?? (unsafeTestValue({})),
|
hooks: args?.hooks ?? (unsafeTestValue({})),
|
||||||
})
|
})
|
||||||
|
const handler = (input: EventInput): Promise<void> => eventHandler(asEventHandlerInput(input))
|
||||||
|
|
||||||
return { handler, abortCalls, promptCalls }
|
return { handler, abortCalls, promptCalls, promptAsyncCalls }
|
||||||
}
|
}
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
@@ -139,6 +166,207 @@ describe("createEventHandler - model fallback", () => {
|
|||||||
expect(promptCalls).toEqual([sessionID])
|
expect(promptCalls).toEqual([sessionID])
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test("does not dispatch duplicate fallback continuations when error events overlap", async () => {
|
||||||
|
//#given
|
||||||
|
const sessionID = "ses_model_fallback_concurrent_events"
|
||||||
|
setMainSession(sessionID)
|
||||||
|
let releasePromptAsync: (() => void) | undefined
|
||||||
|
const promptAsyncBlocked = new Promise<void>((resolve) => {
|
||||||
|
releasePromptAsync = resolve
|
||||||
|
})
|
||||||
|
let firstPromptAsyncStartedResolve: (() => void) | undefined
|
||||||
|
const firstPromptAsyncStarted = new Promise<void>((resolve) => {
|
||||||
|
firstPromptAsyncStartedResolve = resolve
|
||||||
|
})
|
||||||
|
let pendingFallbackArms = 0
|
||||||
|
const modelFallback = unsafeTestValue({
|
||||||
|
setSessionFallbackChain: () => {},
|
||||||
|
setPendingModelFallback: () => {
|
||||||
|
pendingFallbackArms += 1
|
||||||
|
return true
|
||||||
|
},
|
||||||
|
})
|
||||||
|
const { handler, abortCalls, promptAsyncCalls } = createHandler({
|
||||||
|
hooks: { modelFallback },
|
||||||
|
promptAsync: async () => {
|
||||||
|
if (promptAsyncCalls.length === 1) {
|
||||||
|
firstPromptAsyncStartedResolve?.()
|
||||||
|
}
|
||||||
|
await promptAsyncBlocked
|
||||||
|
return {}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
const assistantError = {
|
||||||
|
name: "APIError",
|
||||||
|
data: {
|
||||||
|
message:
|
||||||
|
"Bad Gateway: {\"error\":{\"message\":\"unknown provider for model claude-opus-4-7-thinking\"}}",
|
||||||
|
isRetryable: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
//#when
|
||||||
|
const messageUpdated = handler({
|
||||||
|
event: {
|
||||||
|
type: "message.updated",
|
||||||
|
properties: {
|
||||||
|
info: {
|
||||||
|
id: "msg_err_concurrent_1",
|
||||||
|
sessionID,
|
||||||
|
role: "assistant",
|
||||||
|
error: assistantError,
|
||||||
|
modelID: "claude-opus-4-7-thinking",
|
||||||
|
providerID: "anthropic",
|
||||||
|
agent: "Sisyphus - Ultraworker",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
await firstPromptAsyncStarted
|
||||||
|
const sessionError = handler({
|
||||||
|
event: {
|
||||||
|
type: "session.error",
|
||||||
|
properties: {
|
||||||
|
sessionID,
|
||||||
|
providerID: "anthropic",
|
||||||
|
modelID: "claude-opus-4-7-thinking",
|
||||||
|
error: assistantError,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
releasePromptAsync?.()
|
||||||
|
await Promise.all([messageUpdated, sessionError])
|
||||||
|
|
||||||
|
//#then
|
||||||
|
expect(pendingFallbackArms).toBe(1)
|
||||||
|
expect(promptAsyncCalls).toEqual([sessionID])
|
||||||
|
expect(abortCalls).toEqual([sessionID])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("does not dispatch duplicate fallback continuations when session.error omits provider after dispatch", async () => {
|
||||||
|
//#given
|
||||||
|
const sessionID = "ses_model_fallback_providerless_duplicate"
|
||||||
|
setMainSession(sessionID)
|
||||||
|
let pendingFallbackArms = 0
|
||||||
|
const modelFallback = unsafeTestValue({
|
||||||
|
setSessionFallbackChain: () => {},
|
||||||
|
setPendingModelFallback: () => {
|
||||||
|
pendingFallbackArms += 1
|
||||||
|
return true
|
||||||
|
},
|
||||||
|
})
|
||||||
|
const { handler, abortCalls, promptAsyncCalls } = createHandler({
|
||||||
|
hooks: { modelFallback },
|
||||||
|
promptAsync: async () => ({}),
|
||||||
|
})
|
||||||
|
|
||||||
|
const assistantError = {
|
||||||
|
name: "APIError",
|
||||||
|
data: {
|
||||||
|
message:
|
||||||
|
"Bad Gateway: {\"error\":{\"message\":\"unknown provider for model claude-opus-4-7-thinking\"}}",
|
||||||
|
isRetryable: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
await handler({
|
||||||
|
event: {
|
||||||
|
type: "message.updated",
|
||||||
|
properties: {
|
||||||
|
info: {
|
||||||
|
id: "msg_err_providerless_duplicate_1",
|
||||||
|
sessionID,
|
||||||
|
role: "assistant",
|
||||||
|
error: assistantError,
|
||||||
|
modelID: "claude-opus-4-7-thinking",
|
||||||
|
providerID: "anthropic",
|
||||||
|
agent: "Sisyphus - Ultraworker",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
//#when - same failed model arrives without provider metadata after first dispatch resolved
|
||||||
|
await handler({
|
||||||
|
event: {
|
||||||
|
type: "session.error",
|
||||||
|
properties: {
|
||||||
|
sessionID,
|
||||||
|
error: assistantError,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
//#then
|
||||||
|
expect(pendingFallbackArms).toBe(1)
|
||||||
|
expect(promptAsyncCalls).toEqual([sessionID])
|
||||||
|
expect(abortCalls).toEqual([sessionID])
|
||||||
|
})
|
||||||
|
|
||||||
|
test("does not collapse fallback continuations for different providers with the same model id", async () => {
|
||||||
|
//#given
|
||||||
|
const sessionID = "ses_model_fallback_same_model_different_provider"
|
||||||
|
setMainSession(sessionID)
|
||||||
|
let pendingFallbackArms = 0
|
||||||
|
const modelFallback = unsafeTestValue({
|
||||||
|
setSessionFallbackChain: () => {},
|
||||||
|
setPendingModelFallback: () => {
|
||||||
|
pendingFallbackArms += 1
|
||||||
|
return true
|
||||||
|
},
|
||||||
|
})
|
||||||
|
const { handler, abortCalls, promptAsyncCalls } = createHandler({
|
||||||
|
hooks: { modelFallback },
|
||||||
|
promptAsync: async () => ({}),
|
||||||
|
})
|
||||||
|
|
||||||
|
const assistantError = {
|
||||||
|
name: "APIError",
|
||||||
|
data: {
|
||||||
|
message:
|
||||||
|
"Bad Gateway: {\"error\":{\"message\":\"unknown provider for model claude-opus-4-7-thinking\"}}",
|
||||||
|
isRetryable: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
await handler({
|
||||||
|
event: {
|
||||||
|
type: "message.updated",
|
||||||
|
properties: {
|
||||||
|
info: {
|
||||||
|
id: "msg_err_same_model_provider_1",
|
||||||
|
sessionID,
|
||||||
|
role: "assistant",
|
||||||
|
error: assistantError,
|
||||||
|
modelID: "claude-opus-4-7-thinking",
|
||||||
|
providerID: "anthropic",
|
||||||
|
agent: "Sisyphus - Ultraworker",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
//#when - a distinct provider reports the same normalized model id before idle cleanup
|
||||||
|
await handler({
|
||||||
|
event: {
|
||||||
|
type: "session.error",
|
||||||
|
properties: {
|
||||||
|
sessionID,
|
||||||
|
providerID: "quotio",
|
||||||
|
modelID: "claude-opus-4-7-thinking",
|
||||||
|
error: assistantError,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
//#then
|
||||||
|
expect(pendingFallbackArms).toBe(2)
|
||||||
|
expect(promptAsyncCalls).toEqual([sessionID, sessionID])
|
||||||
|
expect(abortCalls).toEqual([sessionID, sessionID])
|
||||||
|
})
|
||||||
|
|
||||||
test("triggers retry prompt on session.status retry events and applies fallback", async () => {
|
test("triggers retry prompt on session.status retry events and applies fallback", async () => {
|
||||||
//#given
|
//#given
|
||||||
const sessionID = "ses_status_retry_fallback"
|
const sessionID = "ses_status_retry_fallback"
|
||||||
@@ -208,7 +436,7 @@ describe("createEventHandler - model fallback", () => {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
const output = { message: {}, parts: [] as Array<{ type: string; text?: string }> }
|
const output: ChatMessageOutput = { message: {}, parts: [] }
|
||||||
await chatMessageHandler(
|
await chatMessageHandler(
|
||||||
{
|
{
|
||||||
sessionID,
|
sessionID,
|
||||||
@@ -289,7 +517,7 @@ describe("createEventHandler - model fallback", () => {
|
|||||||
expect(promptCalls).toEqual([sessionID])
|
expect(promptCalls).toEqual([sessionID])
|
||||||
})
|
})
|
||||||
|
|
||||||
test("does not re-arm fallback when a duplicate error reports the same failed model after fallback was applied", async () => {
|
test("does not leave stale pending fallback when a providerless duplicate arrives after fallback was applied", async () => {
|
||||||
//#given
|
//#given
|
||||||
const sessionID = "ses_model_fallback_duplicate_surface"
|
const sessionID = "ses_model_fallback_duplicate_surface"
|
||||||
setMainSession(sessionID)
|
setMainSession(sessionID)
|
||||||
@@ -344,7 +572,7 @@ describe("createEventHandler - model fallback", () => {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
const output = { message: {}, parts: [] as Array<{ type: string; text?: string }> }
|
const output: ChatMessageOutput = { message: {}, parts: [] }
|
||||||
await chatMessageHandler(
|
await chatMessageHandler(
|
||||||
{
|
{
|
||||||
sessionID,
|
sessionID,
|
||||||
@@ -354,14 +582,12 @@ describe("createEventHandler - model fallback", () => {
|
|||||||
output,
|
output,
|
||||||
)
|
)
|
||||||
|
|
||||||
//#when - same failed model arrives again through another OpenCode event surface
|
//#when - same failed model arrives again without provider metadata after fallback was applied
|
||||||
await handler({
|
await handler({
|
||||||
event: {
|
event: {
|
||||||
type: "session.error",
|
type: "session.error",
|
||||||
properties: {
|
properties: {
|
||||||
sessionID,
|
sessionID,
|
||||||
providerID: "anthropic",
|
|
||||||
modelID: "claude-opus-4-7-thinking",
|
|
||||||
error: {
|
error: {
|
||||||
name: "UnknownError",
|
name: "UnknownError",
|
||||||
data: {
|
data: {
|
||||||
@@ -375,9 +601,21 @@ describe("createEventHandler - model fallback", () => {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const staleOutput: ChatMessageOutput = { message: {}, parts: [] }
|
||||||
|
await chatMessageHandler(
|
||||||
|
{
|
||||||
|
sessionID,
|
||||||
|
agent: "sisyphus",
|
||||||
|
model: { providerID: "opencode-go", modelID: "kimi-k2.6" },
|
||||||
|
},
|
||||||
|
staleOutput,
|
||||||
|
)
|
||||||
|
|
||||||
//#then
|
//#then
|
||||||
expect(abortCalls).toEqual([sessionID])
|
expect(abortCalls).toEqual([sessionID])
|
||||||
expect(promptCalls).toEqual([sessionID])
|
expect(promptCalls).toEqual([sessionID])
|
||||||
|
expect(modelFallback.hasPendingModelFallback(sessionID)).toBe(false)
|
||||||
|
expect(staleOutput.message["model"]).toBeUndefined()
|
||||||
})
|
})
|
||||||
|
|
||||||
test("does not trigger model-fallback from session.status when runtime_fallback is enabled", async () => {
|
test("does not trigger model-fallback from session.status when runtime_fallback is enabled", async () => {
|
||||||
@@ -509,7 +747,7 @@ describe("createEventHandler - model fallback", () => {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
const output = { message: {}, parts: [] as Array<{ type: string; text?: string }> }
|
const output: ChatMessageOutput = { message: {}, parts: [] }
|
||||||
await chatMessageHandler(
|
await chatMessageHandler(
|
||||||
{
|
{
|
||||||
sessionID,
|
sessionID,
|
||||||
@@ -603,7 +841,7 @@ describe("createEventHandler - model fallback", () => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
const triggerRetryCycle = async (providerID: string, modelID: string) => {
|
const triggerRetryCycle = async (providerID: string, modelID: string) => {
|
||||||
await eventHandler({
|
await eventHandler(asEventHandlerInput({
|
||||||
event: {
|
event: {
|
||||||
type: "session.error",
|
type: "session.error",
|
||||||
properties: {
|
properties: {
|
||||||
@@ -621,9 +859,9 @@ describe("createEventHandler - model fallback", () => {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
})
|
}))
|
||||||
|
|
||||||
const output = { message: {}, parts: [] as Array<{ type: string; text?: string }> }
|
const output: ChatMessageOutput = { message: {}, parts: [] }
|
||||||
await chatMessageHandler(
|
await chatMessageHandler(
|
||||||
{
|
{
|
||||||
sessionID,
|
sessionID,
|
||||||
|
|||||||
+212
-92
@@ -54,6 +54,24 @@ type FirstMessageVariantGate = {
|
|||||||
clear: (sessionID: string) => void;
|
clear: (sessionID: string) => void;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type FallbackContinuationDedupeKeys = {
|
||||||
|
modelKey?: string;
|
||||||
|
providerModelKey?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type FallbackContinuationDedupeState = {
|
||||||
|
modelKeys: Set<string>;
|
||||||
|
providerModelKeys: Set<string>;
|
||||||
|
providerlessModelKeys: Set<string>;
|
||||||
|
};
|
||||||
|
|
||||||
|
type FallbackContinuationContext = {
|
||||||
|
agentName?: string;
|
||||||
|
providerID?: string;
|
||||||
|
dedupeProviderID?: string;
|
||||||
|
modelID?: string;
|
||||||
|
};
|
||||||
|
|
||||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
return typeof value === "object" && value !== null;
|
return typeof value === "object" && value !== null;
|
||||||
}
|
}
|
||||||
@@ -219,6 +237,8 @@ export function createEventHandler(args: {
|
|||||||
const lastHandledModelErrorMessageID = new Map<string, string>();
|
const lastHandledModelErrorMessageID = new Map<string, string>();
|
||||||
const lastHandledRetryStatusKey = new Map<string, string>();
|
const lastHandledRetryStatusKey = new Map<string, string>();
|
||||||
const lastKnownModelBySession = new Map<string, { providerID: string; modelID: string }>();
|
const lastKnownModelBySession = new Map<string, { providerID: string; modelID: string }>();
|
||||||
|
const modelFallbackContinuationsInFlight = new Set<string>();
|
||||||
|
const lastDispatchedModelFallbackContinuationKeys = new Map<string, FallbackContinuationDedupeState>();
|
||||||
|
|
||||||
const resolveFallbackProviderID = (sessionID: string, providerHint?: string): string => {
|
const resolveFallbackProviderID = (sessionID: string, providerHint?: string): string => {
|
||||||
const normalizedProviderHint = providerHint?.trim();
|
const normalizedProviderHint = providerHint?.trim();
|
||||||
@@ -359,55 +379,147 @@ export function createEventHandler(args: {
|
|||||||
return true;
|
return true;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const getFallbackContinuationKeys = (fallbackContext?: FallbackContinuationContext): FallbackContinuationDedupeKeys => {
|
||||||
|
const agentKey = fallbackContext?.agentName
|
||||||
|
? getAgentConfigKey(fallbackContext.agentName).trim().toLowerCase()
|
||||||
|
: "";
|
||||||
|
const providerID = fallbackContext?.dedupeProviderID?.trim().toLowerCase() ?? "";
|
||||||
|
const modelID = fallbackContext?.modelID?.trim().toLowerCase() ?? "";
|
||||||
|
|
||||||
|
if (!agentKey || !modelID) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
modelKey: `${agentKey}:${modelID}`,
|
||||||
|
...(providerID ? { providerModelKey: `${agentKey}:${providerID}:${modelID}` } : {}),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const getFallbackContinuationDedupeState = (sessionID: string): FallbackContinuationDedupeState => {
|
||||||
|
const existingState = lastDispatchedModelFallbackContinuationKeys.get(sessionID);
|
||||||
|
if (existingState) {
|
||||||
|
return existingState;
|
||||||
|
}
|
||||||
|
|
||||||
|
const state = {
|
||||||
|
modelKeys: new Set<string>(),
|
||||||
|
providerModelKeys: new Set<string>(),
|
||||||
|
providerlessModelKeys: new Set<string>(),
|
||||||
|
};
|
||||||
|
lastDispatchedModelFallbackContinuationKeys.set(sessionID, state);
|
||||||
|
return state;
|
||||||
|
};
|
||||||
|
|
||||||
|
const wasFallbackContinuationAlreadyDispatched = (
|
||||||
|
state: FallbackContinuationDedupeState | undefined,
|
||||||
|
keys: FallbackContinuationDedupeKeys,
|
||||||
|
): boolean => {
|
||||||
|
if (!state || !keys.modelKey) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!keys.providerModelKey) {
|
||||||
|
return state.modelKeys.has(keys.modelKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
return state.providerModelKeys.has(keys.providerModelKey) || state.providerlessModelKeys.has(keys.modelKey);
|
||||||
|
};
|
||||||
|
|
||||||
|
const shouldSkipFallbackContinuation = (
|
||||||
|
sessionID: string,
|
||||||
|
source: string,
|
||||||
|
fallbackContext?: FallbackContinuationContext,
|
||||||
|
): boolean => {
|
||||||
|
const fallbackKeys = getFallbackContinuationKeys(fallbackContext);
|
||||||
|
|
||||||
|
if (modelFallbackContinuationsInFlight.has(sessionID)) {
|
||||||
|
log("[event] model-fallback continuation skipped because one is already in flight", { sessionID, source });
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
const lastDispatchedKeys = lastDispatchedModelFallbackContinuationKeys.get(sessionID);
|
||||||
|
if (wasFallbackContinuationAlreadyDispatched(lastDispatchedKeys, fallbackKeys)) {
|
||||||
|
log("[event] model-fallback continuation skipped because matching fallback was already dispatched", {
|
||||||
|
sessionID,
|
||||||
|
source,
|
||||||
|
});
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
const autoContinueAfterFallback = async (
|
const autoContinueAfterFallback = async (
|
||||||
sessionID: string,
|
sessionID: string,
|
||||||
source: string,
|
source: string,
|
||||||
fallbackContext?: {
|
fallbackContext?: FallbackContinuationContext,
|
||||||
agentName?: string;
|
|
||||||
providerID?: string;
|
|
||||||
modelID?: string;
|
|
||||||
},
|
|
||||||
): Promise<void> => {
|
): Promise<void> => {
|
||||||
await pluginContext.client.session.abort({ path: { id: sessionID } }).catch((error) => {
|
const fallbackKeys = getFallbackContinuationKeys(fallbackContext);
|
||||||
log("[event] model-fallback abort failed", { sessionID, source, error });
|
|
||||||
});
|
|
||||||
|
|
||||||
const launchAgent = fallbackContext?.agentName
|
if (shouldSkipFallbackContinuation(sessionID, source, fallbackContext)) {
|
||||||
? resolveRegisteredAgentName(fallbackContext.agentName)
|
|
||||||
: undefined;
|
|
||||||
const launchModel = fallbackContext?.providerID && fallbackContext?.modelID
|
|
||||||
? { providerID: fallbackContext.providerID, modelID: fallbackContext.modelID }
|
|
||||||
: undefined;
|
|
||||||
|
|
||||||
const agentConfigKey = fallbackContext?.agentName
|
|
||||||
? getAgentConfigKey(fallbackContext.agentName)
|
|
||||||
: undefined;
|
|
||||||
const agentSettings = agentConfigKey
|
|
||||||
? pluginConfig.agents?.[agentConfigKey as keyof NonNullable<typeof pluginConfig.agents>]
|
|
||||||
: undefined;
|
|
||||||
const launchVariant = (agentSettings as { variant?: string } | undefined)?.variant;
|
|
||||||
|
|
||||||
const promptBody = {
|
|
||||||
path: { id: sessionID },
|
|
||||||
body: {
|
|
||||||
...(launchAgent ? { agent: launchAgent } : {}),
|
|
||||||
...(launchModel ? { model: launchModel } : {}),
|
|
||||||
...(launchVariant ? { variant: launchVariant } : {}),
|
|
||||||
parts: [createInternalAgentContinuationTextPart("continue")],
|
|
||||||
},
|
|
||||||
query: { directory: pluginContext.directory },
|
|
||||||
};
|
|
||||||
|
|
||||||
if (typeof pluginContext.client.session.promptAsync === "function") {
|
|
||||||
await pluginContext.client.session.promptAsync(promptBody).catch((error) => {
|
|
||||||
log("[event] model-fallback promptAsync failed", { sessionID, source, error });
|
|
||||||
});
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
await pluginContext.client.session.prompt(promptBody).catch((error) => {
|
modelFallbackContinuationsInFlight.add(sessionID);
|
||||||
log("[event] model-fallback prompt failed", { sessionID, source, error });
|
let dispatched = false;
|
||||||
});
|
try {
|
||||||
|
await pluginContext.client.session.abort({ path: { id: sessionID } }).catch((error) => {
|
||||||
|
log("[event] model-fallback abort failed", { sessionID, source, error });
|
||||||
|
});
|
||||||
|
|
||||||
|
const launchAgent = fallbackContext?.agentName
|
||||||
|
? resolveRegisteredAgentName(fallbackContext.agentName)
|
||||||
|
: undefined;
|
||||||
|
const launchModel = fallbackContext?.providerID && fallbackContext?.modelID
|
||||||
|
? { providerID: fallbackContext.providerID, modelID: fallbackContext.modelID }
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
const agentConfigKey = fallbackContext?.agentName
|
||||||
|
? getAgentConfigKey(fallbackContext.agentName)
|
||||||
|
: undefined;
|
||||||
|
const agentSettings = agentConfigKey
|
||||||
|
? pluginConfig.agents?.[agentConfigKey as keyof NonNullable<typeof pluginConfig.agents>]
|
||||||
|
: undefined;
|
||||||
|
const launchVariant = (agentSettings as { variant?: string } | undefined)?.variant;
|
||||||
|
|
||||||
|
const promptBody = {
|
||||||
|
path: { id: sessionID },
|
||||||
|
body: {
|
||||||
|
...(launchAgent ? { agent: launchAgent } : {}),
|
||||||
|
...(launchModel ? { model: launchModel } : {}),
|
||||||
|
...(launchVariant ? { variant: launchVariant } : {}),
|
||||||
|
parts: [createInternalAgentContinuationTextPart("continue")],
|
||||||
|
},
|
||||||
|
query: { directory: pluginContext.directory },
|
||||||
|
};
|
||||||
|
|
||||||
|
if (typeof pluginContext.client.session.promptAsync === "function") {
|
||||||
|
await pluginContext.client.session.promptAsync(promptBody).then(() => {
|
||||||
|
dispatched = true;
|
||||||
|
}).catch((error) => {
|
||||||
|
log("[event] model-fallback promptAsync failed", { sessionID, source, error });
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await pluginContext.client.session.prompt(promptBody).then(() => {
|
||||||
|
dispatched = true;
|
||||||
|
}).catch((error) => {
|
||||||
|
log("[event] model-fallback prompt failed", { sessionID, source, error });
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
if (dispatched && fallbackKeys.modelKey) {
|
||||||
|
const dispatchedKeys = getFallbackContinuationDedupeState(sessionID);
|
||||||
|
dispatchedKeys.modelKeys.add(fallbackKeys.modelKey);
|
||||||
|
if (fallbackKeys.providerModelKey) {
|
||||||
|
dispatchedKeys.providerModelKeys.add(fallbackKeys.providerModelKey);
|
||||||
|
} else {
|
||||||
|
dispatchedKeys.providerlessModelKeys.add(fallbackKeys.modelKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
modelFallbackContinuationsInFlight.delete(sessionID);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
return async (input): Promise<void> => {
|
return async (input): Promise<void> => {
|
||||||
@@ -526,6 +638,8 @@ export function createEventHandler(args: {
|
|||||||
lastHandledModelErrorMessageID.delete(sessionID);
|
lastHandledModelErrorMessageID.delete(sessionID);
|
||||||
lastHandledRetryStatusKey.delete(sessionID);
|
lastHandledRetryStatusKey.delete(sessionID);
|
||||||
lastKnownModelBySession.delete(sessionID);
|
lastKnownModelBySession.delete(sessionID);
|
||||||
|
modelFallbackContinuationsInFlight.delete(sessionID);
|
||||||
|
lastDispatchedModelFallbackContinuationKeys.delete(sessionID);
|
||||||
if (modelFallback) {
|
if (modelFallback) {
|
||||||
clearPendingModelFallback(modelFallback, sessionID);
|
clearPendingModelFallback(modelFallback, sessionID);
|
||||||
clearSessionFallbackChain(modelFallback, sessionID);
|
clearSessionFallbackChain(modelFallback, sessionID);
|
||||||
@@ -643,29 +757,30 @@ export function createEventHandler(args: {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (agentName) {
|
if (agentName) {
|
||||||
const currentProvider = resolveFallbackProviderID(
|
const providerHint = info?.providerID as string | undefined;
|
||||||
sessionID,
|
const currentProvider = resolveFallbackProviderID(sessionID, providerHint);
|
||||||
info?.providerID as string | undefined,
|
|
||||||
);
|
|
||||||
const rawModel = (info?.modelID as string | undefined) ?? "claude-opus-4-7";
|
const rawModel = (info?.modelID as string | undefined) ?? "claude-opus-4-7";
|
||||||
const currentModel = normalizeFallbackModelID(rawModel);
|
const currentModel = normalizeFallbackModelID(rawModel);
|
||||||
applyUserConfiguredFallbackChain(modelFallback, sessionID, agentName, currentProvider, args.pluginConfig);
|
const fallbackContext = {
|
||||||
|
agentName,
|
||||||
|
providerID: currentProvider,
|
||||||
|
dedupeProviderID: providerHint,
|
||||||
|
modelID: currentModel,
|
||||||
|
};
|
||||||
|
const shouldAutoContinue = shouldAutoRetrySession(sessionID) &&
|
||||||
|
!hooks.stopContinuationGuard?.isStopped(sessionID);
|
||||||
|
|
||||||
const setFallback = modelFallback
|
if (!shouldAutoContinue || !shouldSkipFallbackContinuation(sessionID, "message.updated", fallbackContext)) {
|
||||||
? setPendingModelFallback(modelFallback, sessionID, agentName, currentProvider, currentModel)
|
applyUserConfiguredFallbackChain(modelFallback, sessionID, agentName, currentProvider, args.pluginConfig);
|
||||||
: false;
|
|
||||||
|
|
||||||
if (
|
const setFallback = modelFallback
|
||||||
setFallback &&
|
? setPendingModelFallback(modelFallback, sessionID, agentName, currentProvider, currentModel)
|
||||||
shouldAutoRetrySession(sessionID) &&
|
: false;
|
||||||
!hooks.stopContinuationGuard?.isStopped(sessionID)
|
|
||||||
) {
|
if (setFallback && shouldAutoContinue) {
|
||||||
lastHandledModelErrorMessageID.set(sessionID, assistantMessageID);
|
lastHandledModelErrorMessageID.set(sessionID, assistantMessageID);
|
||||||
await autoContinueAfterFallback(sessionID, "message.updated", {
|
await autoContinueAfterFallback(sessionID, "message.updated", fallbackContext);
|
||||||
agentName,
|
}
|
||||||
providerID: currentProvider,
|
|
||||||
modelID: currentModel,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -684,6 +799,7 @@ export function createEventHandler(args: {
|
|||||||
// (non-retry idle) so future failures with the same key can trigger fallback again.
|
// (non-retry idle) so future failures with the same key can trigger fallback again.
|
||||||
if (sessionID && status?.type === "idle") {
|
if (sessionID && status?.type === "idle") {
|
||||||
lastHandledRetryStatusKey.delete(sessionID);
|
lastHandledRetryStatusKey.delete(sessionID);
|
||||||
|
lastDispatchedModelFallbackContinuationKeys.delete(sessionID);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (sessionID && status?.type === "retry" && isModelFallbackEnabled && !isRuntimeFallbackEnabled) {
|
if (sessionID && status?.type === "retry" && isModelFallbackEnabled && !isRuntimeFallbackEnabled) {
|
||||||
@@ -718,22 +834,25 @@ export function createEventHandler(args: {
|
|||||||
const currentProvider = resolveFallbackProviderID(sessionID, parsed.providerID);
|
const currentProvider = resolveFallbackProviderID(sessionID, parsed.providerID);
|
||||||
let currentModel = parsed.modelID ?? lastKnown?.modelID ?? "claude-opus-4-7";
|
let currentModel = parsed.modelID ?? lastKnown?.modelID ?? "claude-opus-4-7";
|
||||||
currentModel = normalizeFallbackModelID(currentModel);
|
currentModel = normalizeFallbackModelID(currentModel);
|
||||||
applyUserConfiguredFallbackChain(modelFallback, sessionID, agentName, currentProvider, args.pluginConfig);
|
const fallbackContext = {
|
||||||
|
agentName,
|
||||||
|
providerID: currentProvider,
|
||||||
|
dedupeProviderID: parsed.providerID,
|
||||||
|
modelID: currentModel,
|
||||||
|
};
|
||||||
|
const shouldAutoContinue = shouldAutoRetrySession(sessionID) &&
|
||||||
|
!hooks.stopContinuationGuard?.isStopped(sessionID);
|
||||||
|
|
||||||
const setFallback = modelFallback
|
if (!shouldAutoContinue || !shouldSkipFallbackContinuation(sessionID, "session.status", fallbackContext)) {
|
||||||
? setPendingModelFallback(modelFallback, sessionID, agentName, currentProvider, currentModel)
|
applyUserConfiguredFallbackChain(modelFallback, sessionID, agentName, currentProvider, args.pluginConfig);
|
||||||
: false;
|
|
||||||
|
|
||||||
if (
|
const setFallback = modelFallback
|
||||||
setFallback &&
|
? setPendingModelFallback(modelFallback, sessionID, agentName, currentProvider, currentModel)
|
||||||
shouldAutoRetrySession(sessionID) &&
|
: false;
|
||||||
!hooks.stopContinuationGuard?.isStopped(sessionID)
|
|
||||||
) {
|
if (setFallback && shouldAutoContinue) {
|
||||||
await autoContinueAfterFallback(sessionID, "session.status", {
|
await autoContinueAfterFallback(sessionID, "session.status", fallbackContext);
|
||||||
agentName,
|
}
|
||||||
providerID: currentProvider,
|
|
||||||
modelID: currentModel,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -804,28 +923,29 @@ export function createEventHandler(args: {
|
|||||||
|
|
||||||
if (agentName) {
|
if (agentName) {
|
||||||
const parsed = extractProviderModelFromErrorMessage(errorMessage);
|
const parsed = extractProviderModelFromErrorMessage(errorMessage);
|
||||||
const currentProvider = resolveFallbackProviderID(
|
const providerHint = (props?.providerID as string | undefined) || parsed.providerID;
|
||||||
sessionID,
|
const currentProvider = resolveFallbackProviderID(sessionID, providerHint);
|
||||||
(props?.providerID as string | undefined) || parsed.providerID,
|
|
||||||
);
|
|
||||||
let currentModel = (props?.modelID as string) || parsed.modelID || "claude-opus-4-7";
|
let currentModel = (props?.modelID as string) || parsed.modelID || "claude-opus-4-7";
|
||||||
currentModel = normalizeFallbackModelID(currentModel);
|
currentModel = normalizeFallbackModelID(currentModel);
|
||||||
applyUserConfiguredFallbackChain(modelFallback, sessionID, agentName, currentProvider, args.pluginConfig);
|
const fallbackContext = {
|
||||||
|
agentName,
|
||||||
|
providerID: currentProvider,
|
||||||
|
dedupeProviderID: providerHint,
|
||||||
|
modelID: currentModel,
|
||||||
|
};
|
||||||
|
const shouldAutoContinue = shouldAutoRetrySession(sessionID) &&
|
||||||
|
!hooks.stopContinuationGuard?.isStopped(sessionID);
|
||||||
|
|
||||||
const setFallback = modelFallback
|
if (!shouldAutoContinue || !shouldSkipFallbackContinuation(sessionID, "session.error", fallbackContext)) {
|
||||||
? setPendingModelFallback(modelFallback, sessionID, agentName, currentProvider, currentModel)
|
applyUserConfiguredFallbackChain(modelFallback, sessionID, agentName, currentProvider, args.pluginConfig);
|
||||||
: false;
|
|
||||||
|
|
||||||
if (
|
const setFallback = modelFallback
|
||||||
setFallback &&
|
? setPendingModelFallback(modelFallback, sessionID, agentName, currentProvider, currentModel)
|
||||||
shouldAutoRetrySession(sessionID) &&
|
: false;
|
||||||
!hooks.stopContinuationGuard?.isStopped(sessionID)
|
|
||||||
) {
|
if (setFallback && shouldAutoContinue) {
|
||||||
await autoContinueAfterFallback(sessionID, "session.error", {
|
await autoContinueAfterFallback(sessionID, "session.error", fallbackContext);
|
||||||
agentName,
|
}
|
||||||
providerID: currentProvider,
|
|
||||||
modelID: currentModel,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user