Merge pull request #4051 from code-yeongyu/supersede/3952-first-prompt-watchdog
fix(runtime-fallback): first-prompt watchdog for silently-stuck subagents (supersedes #3952)
This commit is contained in:
@@ -47,3 +47,14 @@ export const RETRYABLE_ERROR_PATTERNS = [
|
|||||||
* Hook name for identification and logging
|
* Hook name for identification and logging
|
||||||
*/
|
*/
|
||||||
export const HOOK_NAME = "runtime-fallback"
|
export const HOOK_NAME = "runtime-fallback"
|
||||||
|
|
||||||
|
/**
|
||||||
|
* First-prompt watchdog: how long to wait for the first sign of progress
|
||||||
|
* (assistant text/reasoning/finish) from a subagent session before assuming
|
||||||
|
* the provider is silently stuck and dispatching the configured fallback.
|
||||||
|
*
|
||||||
|
* Tuned to be longer than typical first-token latency (well under 30s in
|
||||||
|
* practice) yet much shorter than the 30-minute outer poll timeout that
|
||||||
|
* would otherwise be the only safety net.
|
||||||
|
*/
|
||||||
|
export const DEFAULT_FIRST_PROMPT_WATCHDOG_MS = 90_000
|
||||||
|
|||||||
@@ -0,0 +1,339 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, it } from "bun:test"
|
||||||
|
import type { HookDeps, RuntimeFallbackPluginInput } from "./types"
|
||||||
|
import type { AutoRetryHelpers } from "./auto-retry"
|
||||||
|
import { subagentSessions } from "../../features/claude-code-session-state"
|
||||||
|
import { createFirstPromptWatchdog, observeEventForWatchdog, type FirstPromptWatchdog } from "./first-prompt-watchdog"
|
||||||
|
|
||||||
|
// Real timers are unavoidable here (bun:test has no built-in fake-timer API),
|
||||||
|
// so margins are sized generously to survive a loaded CI runner. Specifically:
|
||||||
|
// - SAFE_WAIT_BEFORE_FIRE_MS must be << WATCHDOG_MS so the cancel call lands
|
||||||
|
// before the timer fires even with significant scheduler delay
|
||||||
|
// (margin: WATCHDOG_MS - SAFE_WAIT_BEFORE_FIRE_MS >= 60ms here).
|
||||||
|
// - SAFE_WAIT_AFTER_FIRE_MS must be >> WATCHDOG_MS so we conclusively
|
||||||
|
// observe whether the timer fired (margin: ~2.5x WATCHDOG_MS).
|
||||||
|
const WATCHDOG_MS = 100
|
||||||
|
const SAFE_WAIT_BEFORE_FIRE_MS = 40
|
||||||
|
const SAFE_WAIT_AFTER_FIRE_MS = 250
|
||||||
|
|
||||||
|
function wait(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||||
|
}
|
||||||
|
|
||||||
|
function createContext(): RuntimeFallbackPluginInput {
|
||||||
|
return {
|
||||||
|
client: {
|
||||||
|
session: {
|
||||||
|
abort: async () => ({}),
|
||||||
|
messages: async () => ({ data: [] }),
|
||||||
|
promptAsync: async () => ({}),
|
||||||
|
},
|
||||||
|
tui: {
|
||||||
|
showToast: async () => ({}),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
directory: "/test/dir",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function createDeps(pluginConfig: Record<string, unknown> = {}): HookDeps {
|
||||||
|
return {
|
||||||
|
ctx: createContext(),
|
||||||
|
config: {
|
||||||
|
enabled: true,
|
||||||
|
retry_on_errors: [429, 503, 529],
|
||||||
|
max_fallback_attempts: 3,
|
||||||
|
cooldown_seconds: 60,
|
||||||
|
timeout_seconds: 30,
|
||||||
|
notify_on_fallback: false,
|
||||||
|
},
|
||||||
|
options: undefined,
|
||||||
|
pluginConfig,
|
||||||
|
sessionStates: new Map(),
|
||||||
|
sessionLastAccess: new Map(),
|
||||||
|
sessionRetryInFlight: new Set(),
|
||||||
|
sessionAwaitingFallbackResult: new Set(),
|
||||||
|
sessionFallbackTimeouts: new Map(),
|
||||||
|
sessionStatusRetryKeys: new Map(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
interface RecordedCalls {
|
||||||
|
abort: Array<{ sessionID: string; source: string }>
|
||||||
|
autoRetry: Array<{ sessionID: string; newModel: string; resolvedAgent: string | undefined; source: string }>
|
||||||
|
}
|
||||||
|
|
||||||
|
function createHelpers(calls: RecordedCalls, resolvedAgentName?: string): AutoRetryHelpers {
|
||||||
|
return {
|
||||||
|
abortSessionRequest: async (sessionID: string, source: string) => {
|
||||||
|
calls.abort.push({ sessionID, source })
|
||||||
|
},
|
||||||
|
clearSessionFallbackTimeout: () => {},
|
||||||
|
scheduleSessionFallbackTimeout: () => {},
|
||||||
|
autoRetryWithFallback: async (sessionID, newModel, resolvedAgent, source) => {
|
||||||
|
calls.autoRetry.push({ sessionID, newModel, resolvedAgent, source })
|
||||||
|
},
|
||||||
|
resolveAgentForSessionFromContext: async () => resolvedAgentName,
|
||||||
|
cleanupStaleSessions: () => {},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const AGENT = "sisyphus-junior"
|
||||||
|
const PRIMARY_MODEL = "openai/gpt-5.4-mini"
|
||||||
|
const FALLBACK_MODEL = "anthropic/claude-haiku-4-5"
|
||||||
|
const PLUGIN_CONFIG_WITH_FALLBACK = {
|
||||||
|
agents: {
|
||||||
|
[AGENT]: {
|
||||||
|
model: PRIMARY_MODEL,
|
||||||
|
fallback_models: [{ model: FALLBACK_MODEL }],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("first-prompt-watchdog", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
subagentSessions.clear()
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
subagentSessions.clear()
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#given a subagent stays silent past the threshold and has a fallback configured #when the watchdog fires #then it aborts the in-flight request and dispatches the fallback model", async () => {
|
||||||
|
// given
|
||||||
|
const sessionID = "session-silent-subagent"
|
||||||
|
subagentSessions.add(sessionID)
|
||||||
|
const deps = createDeps(PLUGIN_CONFIG_WITH_FALLBACK)
|
||||||
|
const calls: RecordedCalls = { abort: [], autoRetry: [] }
|
||||||
|
const helpers = createHelpers(calls, AGENT)
|
||||||
|
const watchdog = createFirstPromptWatchdog(deps, helpers, WATCHDOG_MS)
|
||||||
|
|
||||||
|
// when
|
||||||
|
watchdog.onUserMessage(sessionID, PRIMARY_MODEL, AGENT)
|
||||||
|
await wait(SAFE_WAIT_AFTER_FIRE_MS)
|
||||||
|
|
||||||
|
// then
|
||||||
|
expect(calls.abort).toEqual([{ sessionID, source: "first-prompt-watchdog" }])
|
||||||
|
expect(calls.autoRetry).toHaveLength(1)
|
||||||
|
expect(calls.autoRetry[0].sessionID).toBe(sessionID)
|
||||||
|
expect(calls.autoRetry[0].newModel).toBe(FALLBACK_MODEL)
|
||||||
|
expect(calls.autoRetry[0].source).toBe("first-prompt-watchdog")
|
||||||
|
|
||||||
|
watchdog.dispose()
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#given a subagent produces assistant text before the threshold #when progress is observed #then the watchdog is cancelled and no fallback is dispatched", async () => {
|
||||||
|
// given
|
||||||
|
const sessionID = "session-makes-progress"
|
||||||
|
subagentSessions.add(sessionID)
|
||||||
|
const deps = createDeps(PLUGIN_CONFIG_WITH_FALLBACK)
|
||||||
|
const calls: RecordedCalls = { abort: [], autoRetry: [] }
|
||||||
|
const helpers = createHelpers(calls, AGENT)
|
||||||
|
const watchdog = createFirstPromptWatchdog(deps, helpers, WATCHDOG_MS)
|
||||||
|
|
||||||
|
// when
|
||||||
|
watchdog.onUserMessage(sessionID, PRIMARY_MODEL, AGENT)
|
||||||
|
await wait(SAFE_WAIT_BEFORE_FIRE_MS)
|
||||||
|
watchdog.onAssistantProgress(sessionID)
|
||||||
|
await wait(SAFE_WAIT_AFTER_FIRE_MS)
|
||||||
|
|
||||||
|
// then
|
||||||
|
expect(calls.abort).toEqual([])
|
||||||
|
expect(calls.autoRetry).toEqual([])
|
||||||
|
|
||||||
|
watchdog.dispose()
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#given the session is not a subagent #when a user message is observed #then the watchdog never arms and nothing fires", async () => {
|
||||||
|
// given
|
||||||
|
const sessionID = "session-not-a-subagent"
|
||||||
|
// NOT added to subagentSessions
|
||||||
|
const deps = createDeps(PLUGIN_CONFIG_WITH_FALLBACK)
|
||||||
|
const calls: RecordedCalls = { abort: [], autoRetry: [] }
|
||||||
|
const helpers = createHelpers(calls, AGENT)
|
||||||
|
const watchdog = createFirstPromptWatchdog(deps, helpers, WATCHDOG_MS)
|
||||||
|
|
||||||
|
// when
|
||||||
|
watchdog.onUserMessage(sessionID, PRIMARY_MODEL, AGENT)
|
||||||
|
await wait(SAFE_WAIT_AFTER_FIRE_MS)
|
||||||
|
|
||||||
|
// then
|
||||||
|
expect(calls.abort).toEqual([])
|
||||||
|
expect(calls.autoRetry).toEqual([])
|
||||||
|
|
||||||
|
watchdog.dispose()
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#given a subagent reaches a terminal session state before the threshold #when onSessionTerminal is called #then the watchdog is cancelled and no fallback is dispatched", async () => {
|
||||||
|
// given
|
||||||
|
const sessionID = "session-terminated-early"
|
||||||
|
subagentSessions.add(sessionID)
|
||||||
|
const deps = createDeps(PLUGIN_CONFIG_WITH_FALLBACK)
|
||||||
|
const calls: RecordedCalls = { abort: [], autoRetry: [] }
|
||||||
|
const helpers = createHelpers(calls, AGENT)
|
||||||
|
const watchdog = createFirstPromptWatchdog(deps, helpers, WATCHDOG_MS)
|
||||||
|
|
||||||
|
// when
|
||||||
|
watchdog.onUserMessage(sessionID, PRIMARY_MODEL, AGENT)
|
||||||
|
await wait(SAFE_WAIT_BEFORE_FIRE_MS)
|
||||||
|
watchdog.onSessionTerminal(sessionID)
|
||||||
|
await wait(SAFE_WAIT_AFTER_FIRE_MS)
|
||||||
|
|
||||||
|
// then
|
||||||
|
expect(calls.abort).toEqual([])
|
||||||
|
expect(calls.autoRetry).toEqual([])
|
||||||
|
|
||||||
|
watchdog.dispose()
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#given a subagent silent past the threshold with no fallback configured #when the watchdog fires #then it logs but does not abort or dispatch (lets the existing error-event paths handle it if one arrives later)", async () => {
|
||||||
|
// given
|
||||||
|
const sessionID = "session-no-fallback"
|
||||||
|
subagentSessions.add(sessionID)
|
||||||
|
const deps = createDeps({}) // empty pluginConfig → no fallback models
|
||||||
|
const calls: RecordedCalls = { abort: [], autoRetry: [] }
|
||||||
|
const helpers = createHelpers(calls, AGENT)
|
||||||
|
const watchdog = createFirstPromptWatchdog(deps, helpers, WATCHDOG_MS)
|
||||||
|
|
||||||
|
// when
|
||||||
|
watchdog.onUserMessage(sessionID, PRIMARY_MODEL, AGENT)
|
||||||
|
await wait(SAFE_WAIT_AFTER_FIRE_MS)
|
||||||
|
|
||||||
|
// then
|
||||||
|
expect(calls.abort).toEqual([])
|
||||||
|
expect(calls.autoRetry).toEqual([])
|
||||||
|
|
||||||
|
watchdog.dispose()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
interface RecordedWatchdogCalls {
|
||||||
|
user: Array<{ sessionID: string; model?: string; agent?: string }>
|
||||||
|
progress: string[]
|
||||||
|
terminal: string[]
|
||||||
|
}
|
||||||
|
|
||||||
|
function createRecordingWatchdog(calls: RecordedWatchdogCalls): FirstPromptWatchdog {
|
||||||
|
return {
|
||||||
|
onUserMessage(sessionID, model, agent) {
|
||||||
|
calls.user.push({ sessionID, model, agent })
|
||||||
|
},
|
||||||
|
onAssistantProgress(sessionID) {
|
||||||
|
calls.progress.push(sessionID)
|
||||||
|
},
|
||||||
|
onSessionTerminal(sessionID) {
|
||||||
|
calls.terminal.push(sessionID)
|
||||||
|
},
|
||||||
|
dispose() {},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("observeEventForWatchdog", () => {
|
||||||
|
const sessionID = "session-observed"
|
||||||
|
|
||||||
|
function freshCalls(): RecordedWatchdogCalls {
|
||||||
|
return { user: [], progress: [], terminal: [] }
|
||||||
|
}
|
||||||
|
|
||||||
|
it("#given a message.updated event with role=user #when observed #then onUserMessage is called with sessionID/model/agent", () => {
|
||||||
|
const calls = freshCalls()
|
||||||
|
observeEventForWatchdog(
|
||||||
|
{
|
||||||
|
type: "message.updated",
|
||||||
|
properties: { info: { sessionID, role: "user", model: "openai/gpt-5.4-mini", agent: "sisyphus-junior" } },
|
||||||
|
},
|
||||||
|
createRecordingWatchdog(calls),
|
||||||
|
)
|
||||||
|
expect(calls.user).toEqual([{ sessionID, model: "openai/gpt-5.4-mini", agent: "sisyphus-junior" }])
|
||||||
|
expect(calls.progress).toEqual([])
|
||||||
|
expect(calls.terminal).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
["text", { type: "text", text: "hello" }],
|
||||||
|
["reasoning", { type: "reasoning", text: "thinking..." }],
|
||||||
|
["tool", { type: "tool" }],
|
||||||
|
["tool_use", { type: "tool_use", id: "t1", name: "Read" }],
|
||||||
|
["tool_result", { type: "tool_result", tool_use_id: "t1" }],
|
||||||
|
["tool-call", { type: "tool-call" }],
|
||||||
|
["step-start", { type: "step-start" }],
|
||||||
|
["file", { type: "file" }],
|
||||||
|
])("#given a message.updated assistant event whose only part is type=%s #when observed #then onAssistantProgress is called (model is *working*, not silent)", (_label, part) => {
|
||||||
|
const calls = freshCalls()
|
||||||
|
observeEventForWatchdog(
|
||||||
|
{
|
||||||
|
type: "message.updated",
|
||||||
|
properties: { info: { sessionID, role: "assistant" }, parts: [part] },
|
||||||
|
},
|
||||||
|
createRecordingWatchdog(calls),
|
||||||
|
)
|
||||||
|
expect(calls.progress).toEqual([sessionID])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#given a message.updated assistant event with parts: [] and no error/finish #when observed #then no progress is signalled (no activity yet)", () => {
|
||||||
|
const calls = freshCalls()
|
||||||
|
observeEventForWatchdog(
|
||||||
|
{
|
||||||
|
type: "message.updated",
|
||||||
|
properties: { info: { sessionID, role: "assistant" }, parts: [] },
|
||||||
|
},
|
||||||
|
createRecordingWatchdog(calls),
|
||||||
|
)
|
||||||
|
expect(calls.progress).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#given a message.updated assistant event with info.error set #when observed #then onAssistantProgress is called (the existing error-handling path takes over from here)", () => {
|
||||||
|
const calls = freshCalls()
|
||||||
|
observeEventForWatchdog(
|
||||||
|
{
|
||||||
|
type: "message.updated",
|
||||||
|
properties: { info: { sessionID, role: "assistant", error: { name: "RateLimitError", message: "429" } } },
|
||||||
|
},
|
||||||
|
createRecordingWatchdog(calls),
|
||||||
|
)
|
||||||
|
expect(calls.progress).toEqual([sessionID])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#given a message.updated assistant event with info.finish set #when observed #then onAssistantProgress is called", () => {
|
||||||
|
const calls = freshCalls()
|
||||||
|
observeEventForWatchdog(
|
||||||
|
{
|
||||||
|
type: "message.updated",
|
||||||
|
properties: { info: { sessionID, role: "assistant", finish: "stop" } },
|
||||||
|
},
|
||||||
|
createRecordingWatchdog(calls),
|
||||||
|
)
|
||||||
|
expect(calls.progress).toEqual([sessionID])
|
||||||
|
})
|
||||||
|
|
||||||
|
it.each([["session.idle"], ["session.stop"], ["session.deleted"], ["session.error"]])(
|
||||||
|
"#given a %s event #when observed #then onSessionTerminal is called",
|
||||||
|
(eventType) => {
|
||||||
|
const calls = freshCalls()
|
||||||
|
observeEventForWatchdog(
|
||||||
|
{ type: eventType, properties: { sessionID } },
|
||||||
|
createRecordingWatchdog(calls),
|
||||||
|
)
|
||||||
|
expect(calls.terminal).toEqual([sessionID])
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
it("#given a session.deleted event whose sessionID is carried under properties.info.id #when observed #then onSessionTerminal is still called (matches event-handler shape)", () => {
|
||||||
|
const calls = freshCalls()
|
||||||
|
observeEventForWatchdog(
|
||||||
|
{ type: "session.deleted", properties: { info: { id: sessionID } } },
|
||||||
|
createRecordingWatchdog(calls),
|
||||||
|
)
|
||||||
|
expect(calls.terminal).toEqual([sessionID])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#given an unrelated event type #when observed #then no watchdog method is called", () => {
|
||||||
|
const calls = freshCalls()
|
||||||
|
observeEventForWatchdog(
|
||||||
|
{ type: "session.created", properties: { info: { id: sessionID } } },
|
||||||
|
createRecordingWatchdog(calls),
|
||||||
|
)
|
||||||
|
expect(calls.user).toEqual([])
|
||||||
|
expect(calls.progress).toEqual([])
|
||||||
|
expect(calls.terminal).toEqual([])
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,193 @@
|
|||||||
|
import type { HookDeps, RuntimeFallbackTimeout } from "./types"
|
||||||
|
import type { AutoRetryHelpers } from "./auto-retry"
|
||||||
|
import { HOOK_NAME, DEFAULT_FIRST_PROMPT_WATCHDOG_MS } from "./constants"
|
||||||
|
import { log } from "../../shared/logger"
|
||||||
|
import { subagentSessions } from "../../features/claude-code-session-state"
|
||||||
|
import { createFallbackState } from "./fallback-state"
|
||||||
|
import { getFallbackModelsForSession } from "./fallback-models"
|
||||||
|
import { resolveFallbackBootstrapModel } from "./fallback-bootstrap-model"
|
||||||
|
import { dispatchFallbackRetry } from "./fallback-retry-dispatcher"
|
||||||
|
|
||||||
|
const SOURCE = "first-prompt-watchdog"
|
||||||
|
|
||||||
|
declare function setTimeout(callback: () => void | Promise<void>, delay?: number): RuntimeFallbackTimeout
|
||||||
|
declare function clearTimeout(timeout: RuntimeFallbackTimeout): void
|
||||||
|
|
||||||
|
export interface FirstPromptWatchdog {
|
||||||
|
onUserMessage(sessionID: string, model?: string, agent?: string): void
|
||||||
|
onAssistantProgress(sessionID: string): void
|
||||||
|
onSessionTerminal(sessionID: string): void
|
||||||
|
dispose(): void
|
||||||
|
}
|
||||||
|
|
||||||
|
const TERMINAL_EVENT_TYPES = new Set([
|
||||||
|
"session.idle",
|
||||||
|
"session.stop",
|
||||||
|
"session.deleted",
|
||||||
|
"session.error",
|
||||||
|
])
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Translate an OpenCode session event into the appropriate watchdog signal.
|
||||||
|
*
|
||||||
|
* Progress semantics for cancelling the watchdog:
|
||||||
|
* - assistant `info.error` set: the existing message-update-handler will
|
||||||
|
* deal with the error path; the watchdog has done its job.
|
||||||
|
* - assistant `info.finish` set: the response completed.
|
||||||
|
* - any assistant part with a known type (`text`, `reasoning`, `tool`,
|
||||||
|
* `tool_use`, `tool_result`, `tool-call`, `step-start`, `file`, ...):
|
||||||
|
* the model has started responding. A subagent that immediately runs
|
||||||
|
* tools is *working*, not silent — so any part presence cancels.
|
||||||
|
*/
|
||||||
|
export function observeEventForWatchdog(
|
||||||
|
event: { type: string; properties?: unknown },
|
||||||
|
watchdog: FirstPromptWatchdog,
|
||||||
|
): void {
|
||||||
|
const props = event.properties as Record<string, unknown> | undefined
|
||||||
|
if (!props) return
|
||||||
|
|
||||||
|
if (event.type === "message.updated") {
|
||||||
|
const info = props.info as Record<string, unknown> | undefined
|
||||||
|
const sessionID = info?.sessionID as string | undefined
|
||||||
|
const role = info?.role as string | undefined
|
||||||
|
if (!sessionID || !role) return
|
||||||
|
|
||||||
|
if (role === "user") {
|
||||||
|
const model = info?.model as string | undefined
|
||||||
|
const agent = info?.agent as string | undefined
|
||||||
|
watchdog.onUserMessage(sessionID, model, agent)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if (role === "assistant") {
|
||||||
|
const hasError = info?.error !== undefined
|
||||||
|
const hasFinish = info?.finish !== undefined
|
||||||
|
const eventParts = props.parts as Array<{ type?: string }> | undefined
|
||||||
|
const infoParts = info?.parts as Array<{ type?: string }> | undefined
|
||||||
|
const parts = eventParts ?? infoParts ?? []
|
||||||
|
const hasAnyPart = parts.some((part) => typeof part?.type === "string")
|
||||||
|
if (hasError || hasFinish || hasAnyPart) {
|
||||||
|
watchdog.onAssistantProgress(sessionID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if (TERMINAL_EVENT_TYPES.has(event.type)) {
|
||||||
|
const sessionID =
|
||||||
|
(props.sessionID as string | undefined) ??
|
||||||
|
((props.info as Record<string, unknown> | undefined)?.id as string | undefined)
|
||||||
|
if (sessionID) watchdog.onSessionTerminal(sessionID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createFirstPromptWatchdog(
|
||||||
|
deps: HookDeps,
|
||||||
|
helpers: AutoRetryHelpers,
|
||||||
|
watchdogMs: number = DEFAULT_FIRST_PROMPT_WATCHDOG_MS,
|
||||||
|
): FirstPromptWatchdog {
|
||||||
|
const timers = new Map<string, RuntimeFallbackTimeout>()
|
||||||
|
const armed = new Set<string>()
|
||||||
|
|
||||||
|
const cancel = (sessionID: string): void => {
|
||||||
|
const timer = timers.get(sessionID)
|
||||||
|
if (timer) {
|
||||||
|
clearTimeout(timer)
|
||||||
|
timers.delete(sessionID)
|
||||||
|
}
|
||||||
|
armed.delete(sessionID)
|
||||||
|
}
|
||||||
|
|
||||||
|
const fire = async (sessionID: string, model: string | undefined, agent: string | undefined): Promise<void> => {
|
||||||
|
timers.delete(sessionID)
|
||||||
|
armed.delete(sessionID)
|
||||||
|
|
||||||
|
if (!subagentSessions.has(sessionID)) {
|
||||||
|
log(`[${HOOK_NAME}] ${SOURCE}: session no longer a subagent at fire time, skipping`, { sessionID })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const resolvedAgent = await helpers.resolveAgentForSessionFromContext(sessionID, agent)
|
||||||
|
const fallbackModels = getFallbackModelsForSession(sessionID, resolvedAgent, deps.pluginConfig)
|
||||||
|
|
||||||
|
if (fallbackModels.length === 0) {
|
||||||
|
log(`[${HOOK_NAME}] ${SOURCE}: subagent silent past ${watchdogMs}ms with no fallback configured`, {
|
||||||
|
sessionID,
|
||||||
|
model,
|
||||||
|
agent: resolvedAgent,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
let state = deps.sessionStates.get(sessionID)
|
||||||
|
if (!state) {
|
||||||
|
const initialModel = resolveFallbackBootstrapModel({
|
||||||
|
sessionID,
|
||||||
|
source: SOURCE,
|
||||||
|
eventModel: model,
|
||||||
|
resolvedAgent,
|
||||||
|
pluginConfig: deps.pluginConfig,
|
||||||
|
})
|
||||||
|
if (!initialModel) {
|
||||||
|
log(`[${HOOK_NAME}] ${SOURCE}: no model info available, cannot dispatch fallback`, { sessionID })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
state = createFallbackState(initialModel)
|
||||||
|
deps.sessionStates.set(sessionID, state)
|
||||||
|
deps.sessionLastAccess.set(sessionID, Date.now())
|
||||||
|
}
|
||||||
|
|
||||||
|
log(`[${HOOK_NAME}] ${SOURCE}: subagent silent past ${watchdogMs}ms, dispatching fallback`, {
|
||||||
|
sessionID,
|
||||||
|
model: state.currentModel,
|
||||||
|
fallbackCount: fallbackModels.length,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Unlike the error-event path, the original request is still pending from
|
||||||
|
// OpenCode's perspective when the watchdog fires. Forcefully end it so the
|
||||||
|
// fallback prompt can take over cleanly. Network errors from abort are
|
||||||
|
// logged inside abortSessionRequest and do not block fallback dispatch.
|
||||||
|
await helpers.abortSessionRequest(sessionID, SOURCE)
|
||||||
|
|
||||||
|
await dispatchFallbackRetry(deps, helpers, {
|
||||||
|
sessionID,
|
||||||
|
state,
|
||||||
|
fallbackModels,
|
||||||
|
resolvedAgent,
|
||||||
|
source: SOURCE,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
onUserMessage(sessionID, model, agent) {
|
||||||
|
if (!sessionID) return
|
||||||
|
if (!subagentSessions.has(sessionID)) return
|
||||||
|
if (armed.has(sessionID)) return
|
||||||
|
|
||||||
|
armed.add(sessionID)
|
||||||
|
const timer = setTimeout(async () => {
|
||||||
|
await fire(sessionID, model, agent)
|
||||||
|
}, watchdogMs)
|
||||||
|
timers.set(sessionID, timer)
|
||||||
|
|
||||||
|
log(`[${HOOK_NAME}] ${SOURCE}: armed for subagent`, { sessionID, model, agent, watchdogMs })
|
||||||
|
},
|
||||||
|
onAssistantProgress(sessionID) {
|
||||||
|
if (!sessionID || !armed.has(sessionID)) return
|
||||||
|
cancel(sessionID)
|
||||||
|
log(`[${HOOK_NAME}] ${SOURCE}: cancelled (assistant progress observed)`, { sessionID })
|
||||||
|
},
|
||||||
|
onSessionTerminal(sessionID) {
|
||||||
|
if (!sessionID || !armed.has(sessionID)) return
|
||||||
|
cancel(sessionID)
|
||||||
|
log(`[${HOOK_NAME}] ${SOURCE}: cancelled (session terminal)`, { sessionID })
|
||||||
|
},
|
||||||
|
dispose() {
|
||||||
|
for (const timer of timers.values()) {
|
||||||
|
clearTimeout(timer)
|
||||||
|
}
|
||||||
|
timers.clear()
|
||||||
|
armed.clear()
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@ import { createAutoRetryHelpers } from "./auto-retry"
|
|||||||
import { createChatMessageHandler } from "./chat-message-handler"
|
import { createChatMessageHandler } from "./chat-message-handler"
|
||||||
import { DEFAULT_CONFIG } from "./constants"
|
import { DEFAULT_CONFIG } from "./constants"
|
||||||
import { createEventHandler } from "./event-handler"
|
import { createEventHandler } from "./event-handler"
|
||||||
|
import { createFirstPromptWatchdog, observeEventForWatchdog } from "./first-prompt-watchdog"
|
||||||
import { createMessageUpdateHandler } from "./message-update-handler"
|
import { createMessageUpdateHandler } from "./message-update-handler"
|
||||||
import type { HookDeps, RuntimeFallbackHook, RuntimeFallbackInterval, RuntimeFallbackOptions, RuntimeFallbackPluginInput, RuntimeFallbackTimeout } from "./types"
|
import type { HookDeps, RuntimeFallbackHook, RuntimeFallbackInterval, RuntimeFallbackOptions, RuntimeFallbackPluginInput, RuntimeFallbackTimeout } from "./types"
|
||||||
|
|
||||||
@@ -14,6 +15,7 @@ type RuntimeFallbackHookFactories = {
|
|||||||
createEventHandler: typeof createEventHandler
|
createEventHandler: typeof createEventHandler
|
||||||
createMessageUpdateHandler: typeof createMessageUpdateHandler
|
createMessageUpdateHandler: typeof createMessageUpdateHandler
|
||||||
createChatMessageHandler: typeof createChatMessageHandler
|
createChatMessageHandler: typeof createChatMessageHandler
|
||||||
|
createFirstPromptWatchdog: typeof createFirstPromptWatchdog
|
||||||
}
|
}
|
||||||
|
|
||||||
const defaultRuntimeFallbackHookFactories: RuntimeFallbackHookFactories = {
|
const defaultRuntimeFallbackHookFactories: RuntimeFallbackHookFactories = {
|
||||||
@@ -21,6 +23,7 @@ const defaultRuntimeFallbackHookFactories: RuntimeFallbackHookFactories = {
|
|||||||
createEventHandler,
|
createEventHandler,
|
||||||
createMessageUpdateHandler,
|
createMessageUpdateHandler,
|
||||||
createChatMessageHandler,
|
createChatMessageHandler,
|
||||||
|
createFirstPromptWatchdog,
|
||||||
}
|
}
|
||||||
|
|
||||||
export function createRuntimeFallbackHook(
|
export function createRuntimeFallbackHook(
|
||||||
@@ -59,6 +62,7 @@ export function createRuntimeFallbackHook(
|
|||||||
const baseEventHandler = factories.createEventHandler(deps, helpers)
|
const baseEventHandler = factories.createEventHandler(deps, helpers)
|
||||||
const messageUpdateHandler = factories.createMessageUpdateHandler(deps, helpers)
|
const messageUpdateHandler = factories.createMessageUpdateHandler(deps, helpers)
|
||||||
const chatMessageHandler = factories.createChatMessageHandler(deps)
|
const chatMessageHandler = factories.createChatMessageHandler(deps)
|
||||||
|
const firstPromptWatchdog = factories.createFirstPromptWatchdog(deps, helpers)
|
||||||
|
|
||||||
let cleanupInterval: RuntimeFallbackInterval | null = null
|
let cleanupInterval: RuntimeFallbackInterval | null = null
|
||||||
let intervalStarted = false
|
let intervalStarted = false
|
||||||
@@ -77,6 +81,10 @@ export function createRuntimeFallbackHook(
|
|||||||
const eventHandler = async ({ event }: { event: { type: string; properties?: unknown } }) => {
|
const eventHandler = async ({ event }: { event: { type: string; properties?: unknown } }) => {
|
||||||
ensureInterval()
|
ensureInterval()
|
||||||
|
|
||||||
|
if (config.enabled) {
|
||||||
|
observeEventForWatchdog(event, firstPromptWatchdog)
|
||||||
|
}
|
||||||
|
|
||||||
if (event.type === "message.updated") {
|
if (event.type === "message.updated") {
|
||||||
if (!config.enabled) return
|
if (!config.enabled) return
|
||||||
const props = event.properties as Record<string, unknown> | undefined
|
const props = event.properties as Record<string, unknown> | undefined
|
||||||
@@ -95,6 +103,8 @@ export function createRuntimeFallbackHook(
|
|||||||
clearTimeout(fallbackTimeout)
|
clearTimeout(fallbackTimeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
firstPromptWatchdog.dispose()
|
||||||
|
|
||||||
deps.sessionStates.clear()
|
deps.sessionStates.clear()
|
||||||
deps.sessionLastAccess.clear()
|
deps.sessionLastAccess.clear()
|
||||||
deps.sessionRetryInFlight.clear()
|
deps.sessionRetryInFlight.clear()
|
||||||
|
|||||||
@@ -39,7 +39,18 @@ export function createSessionStatusHandler(
|
|||||||
// retry status message may not contain "retrying in" text alongside the error.
|
// retry status message may not contain "retrying in" text alongside the error.
|
||||||
const messageLower = retryMessage.toLowerCase()
|
const messageLower = retryMessage.toLowerCase()
|
||||||
const matchesRetryablePattern = RETRYABLE_ERROR_PATTERNS.some((pattern) => pattern.test(messageLower))
|
const matchesRetryablePattern = RETRYABLE_ERROR_PATTERNS.some((pattern) => pattern.test(messageLower))
|
||||||
if (!matchesRetryablePattern) return
|
if (!matchesRetryablePattern) {
|
||||||
|
// Diagnostic: capture the actual retry message content so we can extend
|
||||||
|
// RETRYABLE_ERROR_PATTERNS if a provider emits a phrasing we don't yet match.
|
||||||
|
if (retryMessage) {
|
||||||
|
log(`[${HOOK_NAME}] session.status retry with non-matching message`, {
|
||||||
|
sessionID,
|
||||||
|
attempt: status.attempt,
|
||||||
|
retryMessage,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const retryKey = `${extractRetryAttempt(status.attempt, retryMessage)}:${normalizeRetryStatusMessage(retryMessage)}`
|
const retryKey = `${extractRetryAttempt(status.attempt, retryMessage)}:${normalizeRetryStatusMessage(retryMessage)}`
|
||||||
|
|||||||
Reference in New Issue
Block a user