luna/oh-my-opencode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3576 from Disaster-Terminator/fix/background-busy-stall-detection

Browse Source
This commit is contained in:
YeonGyu-Kim
2026-05-15 21:56:31 +09:00
committed by GitHub
parent 27788b4ab0 b6caa5d3e0
commit f54888b2fe
3 changed files with 105 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/features/background-agent/manager.test.ts
+16 -10
View File
@@ -4354,7 +4354,7 @@ describe("BackgroundManager.checkAndInterruptStaleTasks", () => {
expect(task.status).toBe("cancelled")
})
test("should NOT interrupt task when session is running, even with stale lastUpdate", async () => {
test("should interrupt running session when lastUpdate exceeds stale timeout", async () => {
//#given
const client = {
session: {
@@ -4367,6 +4367,7 @@ describe("BackgroundManager.checkAndInterruptStaleTasks", () => {
},
}
const manager = new BackgroundManager({ pluginContext: createPluginInput(client), config: { staleTimeoutMs: 180_000 } })
stubNotifyParentSession(manager)
const task: BackgroundTask = {
id: "task-running-session",
@@ -4386,11 +4387,12 @@ describe("BackgroundManager.checkAndInterruptStaleTasks", () => {
getTaskMap(manager).set(task.id, task)
//#when - session is actively running
//#when - session still reports running, but progress is stale
await manager["checkAndInterruptStaleTasks"]({ "session-running": { type: "running" } })
//#then - task survives because session is running
expect(task.status).toBe("running")
//#then
expect(task.status).toBe("cancelled")
expect(task.error).toContain("Stale timeout")
})
test("should interrupt task when session is idle and lastUpdate exceeds stale timeout", async () => {
@@ -4434,7 +4436,7 @@ describe("BackgroundManager.checkAndInterruptStaleTasks", () => {
expect(task.error).toContain("Stale timeout")
})
test("should NOT interrupt running session even with very old lastUpdate (no safety net)", async () => {
test("should interrupt running session even with very old lastUpdate", async () => {
//#given
const client = {
session: {
@@ -4444,6 +4446,7 @@ describe("BackgroundManager.checkAndInterruptStaleTasks", () => {
},
}
const manager = new BackgroundManager({ pluginContext: createPluginInput(client), config: { staleTimeoutMs: 180_000 } })
stubNotifyParentSession(manager)
const task: BackgroundTask = {
id: "task-long-running",
@@ -4466,11 +4469,12 @@ describe("BackgroundManager.checkAndInterruptStaleTasks", () => {
//#when - session is running, lastUpdate 15min old
await manager["checkAndInterruptStaleTasks"]({ "session-long": { type: "running" } })
//#then - running sessions are NEVER stale-killed
expect(task.status).toBe("running")
//#then
expect(task.status).toBe("cancelled")
expect(task.error).toContain("Stale timeout")
})
test("should NOT interrupt running session with no progress (undefined lastUpdate)", async () => {
test("should interrupt running session with no progress after message staleness timeout", async () => {
//#given - no progress at all, but session is running
const client = {
session: {
@@ -4480,6 +4484,7 @@ describe("BackgroundManager.checkAndInterruptStaleTasks", () => {
},
}
const manager = new BackgroundManager({ pluginContext: createPluginInput(client), config: { messageStalenessTimeoutMs: 600_000 } })
stubNotifyParentSession(manager)
const task: BackgroundTask = {
id: "task-running-no-progress",
@@ -4500,8 +4505,9 @@ describe("BackgroundManager.checkAndInterruptStaleTasks", () => {
//#when - session is running despite no progress
await manager["checkAndInterruptStaleTasks"]({ "session-rnp": { type: "running" } })
//#then - running sessions are NEVER killed
expect(task.status).toBe("running")
//#then
expect(task.status).toBe("cancelled")
expect(task.error).toContain("no activity")
})
test("should interrupt task with no lastUpdate after messageStalenessTimeout", async () => {
src/features/background-agent/task-poller.test.ts
+87 -57
View File
@@ -177,6 +177,7 @@ describe("checkAndInterruptStaleTasks", () => {
//#then
expect(task.status).toBe("cancelled")
expect(task.error).toContain("no activity")
expect(task.error).toContain("messageStalenessTimeoutMs")
})
it("should await abort before resolving for no-progress stale interruption", async () => {
@@ -253,13 +254,13 @@ describe("checkAndInterruptStaleTasks", () => {
expect(task.error).toContain("no activity")
})
it("should NOT interrupt task when session is running, even if lastUpdate exceeds stale timeout", async () => {
//#given - lastUpdate is 5min old but session is actively running
it("should NOT interrupt busy session when progress is within the configured stale timeout", async () => {
//#given - session is busy and progress was observed recently
const task = createRunningTask({
startedAt: new Date(Date.now() - 300_000),
progress: {
toolCalls: 2,
lastUpdate: new Date(Date.now() - 300_000),
lastUpdate: new Date(Date.now() - 60_000),
},
})
@@ -273,12 +274,12 @@ describe("checkAndInterruptStaleTasks", () => {
sessionStatuses: { "ses-1": { type: "busy" } },
})
//#then - task should survive because session is actively busy
//#then
expect(task.status).toBe("running")
})
it("should NOT interrupt busy session task even with very old lastUpdate", async () => {
//#given - lastUpdate is 15min old, but session is still busy
it("should interrupt busy session task when lastUpdate exceeds stale timeout", async () => {
//#given - the session still reports busy, but no progress arrived within the configured timeout
const task = createRunningTask({
startedAt: new Date(Date.now() - 900_000),
progress: {
@@ -297,14 +298,15 @@ describe("checkAndInterruptStaleTasks", () => {
sessionStatuses: { "ses-1": { type: "busy" } },
})
//#then - busy sessions are NEVER stale-killed (babysitter + TTL prune handle these)
expect(task.status).toBe("running")
//#then
expect(task.status).toBe("cancelled")
expect(task.error).toContain("Stale timeout")
})
it("should NOT interrupt busy session even with no progress (undefined lastUpdate)", async () => {
//#given - task has no progress at all, but session is busy
it("should NOT interrupt busy session with no progress within message staleness timeout", async () => {
//#given - task has no progress yet, but it is still inside the configured first-progress window
const task = createRunningTask({
startedAt: new Date(Date.now() - 15 * 60 * 1000),
startedAt: new Date(Date.now() - 5 * 60 * 1000),
progress: undefined,
})
@@ -318,10 +320,33 @@ describe("checkAndInterruptStaleTasks", () => {
sessionStatuses: { "ses-1": { type: "busy" } },
})
//#then - task should survive because session is actively running
//#then
expect(task.status).toBe("running")
})
it("should interrupt busy session when it exceeds configured no-progress timeout", async () => {
//#given - the session reports busy, but no progress event arrived within the configured timeout
const task = createRunningTask({
startedAt: new Date(Date.now() - 15 * 60 * 1000),
progress: undefined,
})
//#when
await checkAndInterruptStaleTasks({
tasks: [task],
client: mockClient as never,
config: { messageStalenessTimeoutMs: 600_000 },
concurrencyManager: mockConcurrencyManager as never,
notifyParentSession: mockNotify,
sessionStatuses: { "ses-1": { type: "busy" } },
})
//#then
expect(task.status).toBe("cancelled")
expect(task.error).toContain("no activity")
expect(mockNotify).toHaveBeenCalledWith(task)
})
it("should interrupt task when session is idle and lastUpdate exceeds stale timeout", async () => {
//#given - lastUpdate is 5min old and session is idle
const task = createRunningTask({
@@ -347,8 +372,8 @@ describe("checkAndInterruptStaleTasks", () => {
expect(task.error).toContain("Stale timeout")
})
it("should NOT interrupt running session task even with very old lastUpdate", async () => {
//#given - lastUpdate is 15min old, but session is still running
it("should interrupt running session task when lastUpdate exceeds stale timeout", async () => {
//#given - the session reports running, but no progress arrived within the configured timeout
const task = createRunningTask({
startedAt: new Date(Date.now() - 900_000),
progress: {
@@ -367,18 +392,19 @@ describe("checkAndInterruptStaleTasks", () => {
sessionStatuses: { "ses-1": { type: "running" } },
})
//#then - running sessions are NEVER stale-killed (babysitter + TTL prune handle these)
expect(task.status).toBe("running")
//#then
expect(task.status).toBe("cancelled")
expect(task.error).toContain("Stale timeout")
})
it("should NOT interrupt running session even with no progress (undefined lastUpdate)", async () => {
//#given - task has no progress at all, but session is running
it("should interrupt running session with no progress after message staleness timeout", async () => {
//#given - the session reports running, but no progress ever arrived within the configured timeout
const task = createRunningTask({
startedAt: new Date(Date.now() - 15 * 60 * 1000),
progress: undefined,
})
//#when — session is running
//#when - session is running
await checkAndInterruptStaleTasks({
tasks: [task],
client: mockClient as never,
@@ -388,12 +414,13 @@ describe("checkAndInterruptStaleTasks", () => {
sessionStatuses: { "ses-1": { type: "running" } },
})
//#then — running sessions are NEVER killed, even without progress
expect(task.status).toBe("running")
//#then
expect(task.status).toBe("cancelled")
expect(task.error).toContain("no activity")
})
it("should NOT cancel healthy task on first missing status poll", async () => {
//#given — one missing poll should not be enough to declare the session gone
//#given - one missing poll should not be enough to declare the session gone
const task = createRunningTask({
startedAt: new Date(Date.now() - 300_000),
progress: {
@@ -419,7 +446,7 @@ describe("checkAndInterruptStaleTasks", () => {
})
it("should NOT cancel task when session.get confirms the session still exists", async () => {
//#given — repeated missing polls but direct lookup still succeeds
//#given - repeated missing polls but direct lookup still succeeds
const task = createRunningTask({
startedAt: new Date(Date.now() - 300_000),
progress: {
@@ -446,7 +473,7 @@ describe("checkAndInterruptStaleTasks", () => {
})
it("should NOT cancel task when session.get returns a transient error response", async () => {
//#given — repeated missing polls but lookup failed with a retryable transport error
//#given - repeated missing polls but lookup failed with a retryable transport error
const task = createRunningTask({
startedAt: new Date(Date.now() - 300_000),
progress: {
@@ -478,7 +505,7 @@ describe("checkAndInterruptStaleTasks", () => {
})
it("should use session-gone timeout when session is missing from status map (with progress)", async () => {
//#given — lastUpdate 2min ago, session completely gone from status
//#given - lastUpdate 2min ago, session completely gone from status
const task = createRunningTask({
startedAt: new Date(Date.now() - 300_000),
progress: {
@@ -490,7 +517,7 @@ describe("checkAndInterruptStaleTasks", () => {
mockClient.session.get.mockRejectedValue(new Error("missing"))
//#when — empty sessionStatuses (session gone), sessionGoneTimeoutMs = 60s
//#when - empty sessionStatuses (session gone), sessionGoneTimeoutMs = 60s
await checkAndInterruptStaleTasks({
tasks: [task],
client: mockClient as never,
@@ -500,7 +527,7 @@ describe("checkAndInterruptStaleTasks", () => {
sessionStatuses: {},
})
//#then — cancelled because session gone timeout (60s) < timeSinceLastUpdate (120s)
//#then - cancelled because session gone timeout (60s) < timeSinceLastUpdate (120s)
expect(task.status).toBe("cancelled")
expect(task.error).toContain("session gone from status registry")
})
@@ -545,7 +572,7 @@ describe("checkAndInterruptStaleTasks", () => {
})
it("should use session-gone timeout when session is missing from status map (no progress)", async () => {
//#given — task started 2min ago, no progress, session completely gone
//#given - task started 2min ago, no progress, session completely gone
const task = createRunningTask({
startedAt: new Date(Date.now() - 120_000),
progress: undefined,
@@ -554,7 +581,7 @@ describe("checkAndInterruptStaleTasks", () => {
mockClient.session.get.mockRejectedValue(new Error("missing"))
//#when — session gone, sessionGoneTimeoutMs = 60s
//#when - session gone, sessionGoneTimeoutMs = 60s
await checkAndInterruptStaleTasks({
tasks: [task],
client: mockClient as never,
@@ -564,13 +591,13 @@ describe("checkAndInterruptStaleTasks", () => {
sessionStatuses: {},
})
//#then — cancelled because session gone timeout (60s) < runtime (120s)
//#then - cancelled because session gone timeout (60s) < runtime (120s)
expect(task.status).toBe("cancelled")
expect(task.error).toContain("session gone from status registry")
})
it("should NOT use session-gone timeout when session is idle (present in status map)", async () => {
//#given — lastUpdate 2min ago, session is idle (present in status but not active)
//#given - lastUpdate 2min ago, session is idle (present in status but not active)
const task = createRunningTask({
startedAt: new Date(Date.now() - 300_000),
progress: {
@@ -582,7 +609,7 @@ describe("checkAndInterruptStaleTasks", () => {
mockClient.session.get.mockRejectedValue(new Error("missing"))
//#when — session is idle (present in map), staleTimeoutMs = 180s
//#when - session is idle (present in map), staleTimeoutMs = 180s
await checkAndInterruptStaleTasks({
tasks: [task],
client: mockClient as never,
@@ -592,12 +619,12 @@ describe("checkAndInterruptStaleTasks", () => {
sessionStatuses: { "ses-1": { type: "idle" } },
})
//#then — still running because normal staleTimeout (180s) > timeSinceLastUpdate (120s)
//#then - still running because normal staleTimeout (180s) > timeSinceLastUpdate (120s)
expect(task.status).toBe("running")
})
it("should use default session-gone timeout when not configured", async () => {
//#given — lastUpdate 2min ago, session gone, no sessionGoneTimeoutMs config
//#given - lastUpdate 2min ago, session gone, no sessionGoneTimeoutMs config
const task = createRunningTask({
startedAt: new Date(Date.now() - 300_000),
progress: {
@@ -609,7 +636,7 @@ describe("checkAndInterruptStaleTasks", () => {
mockClient.session.get.mockRejectedValue(new Error("missing"))
//#when — no config (default sessionGoneTimeoutMs = 60_000)
//#when - no config (default sessionGoneTimeoutMs = 60_000)
await checkAndInterruptStaleTasks({
tasks: [task],
client: mockClient as never,
@@ -619,13 +646,13 @@ describe("checkAndInterruptStaleTasks", () => {
sessionStatuses: {},
})
//#then — cancelled because default session gone timeout (60s) < timeSinceLastUpdate (120s)
//#then - cancelled because default session gone timeout (60s) < timeSinceLastUpdate (120s)
expect(task.status).toBe("cancelled")
expect(task.error).toContain("session gone from status registry")
})
it("should NOT interrupt task when session is busy (OpenCode status), even if lastUpdate exceeds stale timeout", async () => {
//#given — lastUpdate is 5min old but session is "busy" (OpenCode's actual status for active sessions)
it("should interrupt task when busy session exceeds stale timeout", async () => {
//#given - lastUpdate is 5min old and session is still "busy"
const task = createRunningTask({
startedAt: new Date(Date.now() - 300_000),
progress: {
@@ -634,7 +661,7 @@ describe("checkAndInterruptStaleTasks", () => {
},
})
//#when — session status is "busy" (not "running" — OpenCode uses "busy" for active LLM processing)
//#when - session status is "busy" (not "running" - OpenCode uses "busy" for active LLM processing)
await checkAndInterruptStaleTasks({
tasks: [task],
client: mockClient as never,
@@ -644,12 +671,13 @@ describe("checkAndInterruptStaleTasks", () => {
sessionStatuses: { "ses-1": { type: "busy" } },
})
//#then — "busy" sessions must be protected from stale-kill
expect(task.status).toBe("running")
//#then
expect(task.status).toBe("cancelled")
expect(task.error).toContain("Stale timeout")
})
it("should NOT interrupt task when session is in retry state", async () => {
//#given — lastUpdate is 5min old but session is retrying
it("should interrupt task when retry session exceeds stale timeout", async () => {
//#given - lastUpdate is 5min old but session is retrying
const task = createRunningTask({
startedAt: new Date(Date.now() - 300_000),
progress: {
@@ -658,7 +686,7 @@ describe("checkAndInterruptStaleTasks", () => {
},
})
//#when — session status is "retry" (OpenCode retries on transient API errors)
//#when - session status is "retry" (OpenCode retries on transient API errors)
await checkAndInterruptStaleTasks({
tasks: [task],
client: mockClient as never,
@@ -668,18 +696,19 @@ describe("checkAndInterruptStaleTasks", () => {
sessionStatuses: { "ses-1": { type: "retry" } },
})
//#then — retry sessions must be protected from stale-kill
expect(task.status).toBe("running")
//#then
expect(task.status).toBe("cancelled")
expect(task.error).toContain("Stale timeout")
})
it("should NOT interrupt busy session even with no progress (undefined lastUpdate)", async () => {
//#given — no progress at all, session is "busy" (thinking model with no streamed tokens yet)
it("should interrupt busy session with no progress after message staleness timeout", async () => {
//#given - no progress at all, session is still "busy"
const task = createRunningTask({
startedAt: new Date(Date.now() - 15 * 60 * 1000),
progress: undefined,
})
//#when — session is busy
//#when - session is busy
await checkAndInterruptStaleTasks({
tasks: [task],
client: mockClient as never,
@@ -689,8 +718,9 @@ describe("checkAndInterruptStaleTasks", () => {
sessionStatuses: { "ses-1": { type: "busy" } },
})
//#then — busy sessions with no progress must survive
expect(task.status).toBe("running")
//#then
expect(task.status).toBe("cancelled")
expect(task.error).toContain("no activity")
})
it("should release concurrency key when interrupting a never-updated task", async () => {
@@ -742,7 +772,7 @@ describe("checkAndInterruptStaleTasks", () => {
})
it('should NOT protect task when session has terminal non-idle status like "interrupted"', async () => {
//#given — lastUpdate is 5min old, session is "interrupted" (terminal, not active)
//#given - lastUpdate is 5min old, session is "interrupted" (terminal, not active)
const task = createRunningTask({
startedAt: new Date(Date.now() - 300_000),
progress: {
@@ -751,7 +781,7 @@ describe("checkAndInterruptStaleTasks", () => {
},
})
//#when — session status is "interrupted" (terminal)
//#when - session status is "interrupted" (terminal)
await checkAndInterruptStaleTasks({
tasks: [task],
client: mockClient as never,
@@ -761,13 +791,13 @@ describe("checkAndInterruptStaleTasks", () => {
sessionStatuses: { "ses-1": { type: "interrupted" } },
})
//#then — terminal statuses should not protect from stale timeout
//#then - terminal statuses should not protect from stale timeout
expect(task.status).toBe("cancelled")
expect(task.error).toContain("Stale timeout")
})
it('should NOT protect task when session has unknown status type', async () => {
//#given — lastUpdate is 5min old, session has an unknown status
//#given - lastUpdate is 5min old, session has an unknown status
const task = createRunningTask({
startedAt: new Date(Date.now() - 300_000),
progress: {
@@ -776,7 +806,7 @@ describe("checkAndInterruptStaleTasks", () => {
},
})
//#when — session has unknown status type
//#when - session has unknown status type
await checkAndInterruptStaleTasks({
tasks: [task],
client: mockClient as never,
@@ -786,7 +816,7 @@ describe("checkAndInterruptStaleTasks", () => {
sessionStatuses: { "ses-1": { type: "some-weird-status" } },
})
//#then — unknown statuses should not protect from stale timeout
//#then - unknown statuses should not protect from stale timeout
expect(task.status).toBe("cancelled")
expect(task.error).toContain("Stale timeout")
})
src/features/background-agent/task-poller.ts
+2 -4
View File
@@ -19,6 +19,7 @@ import { removeTaskToastTracking } from "./remove-task-toast-tracking"
import { MIN_SESSION_GONE_POLLS, verifySessionExists } from "./session-existence"
import { isActiveSessionStatus } from "./session-status-classifier"
const TERMINAL_TASK_STATUSES = new Set<BackgroundTask["status"]>([
"completed",
"error",
@@ -145,7 +146,6 @@ export async function checkAndInterruptStaleTasks(args: {
if (!startedAt || !sessionID) continue
const sessionStatus = sessionStatuses?.[sessionID]?.type
const sessionIsRunning = sessionStatus !== undefined && isActiveSessionStatus(sessionStatus)
const sessionMissing = sessionStatuses !== undefined && sessionStatus === undefined
const runtime = now - startedAt.getTime()
@@ -160,7 +160,6 @@ export async function checkAndInterruptStaleTasks(args: {
if (!task.progress?.lastUpdate) {
if (shouldSkipInactivityTimeout) continue
if (sessionIsRunning) continue
if (sessionMissing && !sessionGone) continue
const effectiveTimeout = sessionGone ? sessionGoneTimeoutMs : messageStalenessMs
if (runtime <= effectiveTimeout) continue
@@ -173,7 +172,7 @@ export async function checkAndInterruptStaleTasks(args: {
const staleMinutes = Math.round(runtime / 60000)
const reason = sessionGone ? "session gone from status registry" : "no activity"
task.status = "cancelled"
task.error = `Stale timeout (${reason} for ${staleMinutes}min since start). This is a FINAL cancellation - do NOT create a replacement task. If the timeout is too short, increase 'background_task.${sessionGone ? "sessionGoneTimeoutMs" : "staleTimeoutMs"}' in .opencode/${CONFIG_BASENAME}.json.`
task.error = `Stale timeout (${reason} for ${staleMinutes}min since start). This is a FINAL cancellation - do NOT create a replacement task. If the timeout is too short, increase 'background_task.${sessionGone ? "sessionGoneTimeoutMs" : "messageStalenessTimeoutMs"}' in .opencode/${CONFIG_BASENAME}.json.`
task.completedAt = new Date()
if (task.concurrencyKey) {
@@ -194,7 +193,6 @@ export async function checkAndInterruptStaleTasks(args: {
continue
}
if (sessionIsRunning) continue
if (shouldSkipInactivityTimeout) continue
if (runtime < MIN_RUNTIME_BEFORE_STALE_MS) continue