fix(atlas): stop only after 10 consecutive prompt failures
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
@@ -13,6 +13,7 @@ import type { AtlasHookOptions, SessionState } from "./types"
|
|||||||
|
|
||||||
const CONTINUATION_COOLDOWN_MS = 5000
|
const CONTINUATION_COOLDOWN_MS = 5000
|
||||||
const FAILURE_BACKOFF_MS = 5 * 60 * 1000
|
const FAILURE_BACKOFF_MS = 5 * 60 * 1000
|
||||||
|
const MAX_CONSECUTIVE_PROMPT_FAILURES = 10
|
||||||
const RETRY_DELAY_MS = CONTINUATION_COOLDOWN_MS + 1000
|
const RETRY_DELAY_MS = CONTINUATION_COOLDOWN_MS + 1000
|
||||||
|
|
||||||
function hasRunningBackgroundTasks(sessionID: string, options?: AtlasHookOptions): boolean {
|
function hasRunningBackgroundTasks(sessionID: string, options?: AtlasHookOptions): boolean {
|
||||||
@@ -77,7 +78,7 @@ function scheduleRetry(input: {
|
|||||||
sessionState.pendingRetryTimer = setTimeout(async () => {
|
sessionState.pendingRetryTimer = setTimeout(async () => {
|
||||||
sessionState.pendingRetryTimer = undefined
|
sessionState.pendingRetryTimer = undefined
|
||||||
|
|
||||||
if (sessionState.promptFailureCount >= 2) return
|
if (sessionState.promptFailureCount >= MAX_CONSECUTIVE_PROMPT_FAILURES) return
|
||||||
if (sessionState.waitingForFinalWaveApproval) return
|
if (sessionState.waitingForFinalWaveApproval) return
|
||||||
|
|
||||||
const currentBoulder = readBoulderState(ctx.directory)
|
const currentBoulder = readBoulderState(ctx.directory)
|
||||||
@@ -150,7 +151,7 @@ export async function handleAtlasSessionIdle(input: {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if (sessionState.promptFailureCount >= 2) {
|
if (sessionState.promptFailureCount >= MAX_CONSECUTIVE_PROMPT_FAILURES) {
|
||||||
const timeSinceLastFailure =
|
const timeSinceLastFailure =
|
||||||
sessionState.lastFailureAt !== undefined ? now - sessionState.lastFailureAt : Number.POSITIVE_INFINITY
|
sessionState.lastFailureAt !== undefined ? now - sessionState.lastFailureAt : Number.POSITIVE_INFINITY
|
||||||
if (timeSinceLastFailure < FAILURE_BACKOFF_MS) {
|
if (timeSinceLastFailure < FAILURE_BACKOFF_MS) {
|
||||||
|
|||||||
@@ -1746,7 +1746,7 @@ session_id: ses_untrusted_999
|
|||||||
expect(mockInput._promptMock).toHaveBeenCalledTimes(1)
|
expect(mockInput._promptMock).toHaveBeenCalledTimes(1)
|
||||||
})
|
})
|
||||||
|
|
||||||
test("should stop continuation after 2 consecutive prompt failures (issue #1355)", async () => {
|
test("should stop continuation after 10 consecutive prompt failures (issue #1355)", async () => {
|
||||||
//#given - boulder state with incomplete plan and prompt always fails
|
//#given - boulder state with incomplete plan and prompt always fails
|
||||||
const planPath = join(TEST_DIR, "test-plan.md")
|
const planPath = join(TEST_DIR, "test-plan.md")
|
||||||
writeFileSync(planPath, "# Plan\n- [ ] Task 1\n- [ ] Task 2")
|
writeFileSync(planPath, "# Plan\n- [ ] Task 1\n- [ ] Task 2")
|
||||||
@@ -1759,7 +1759,7 @@ session_id: ses_untrusted_999
|
|||||||
}
|
}
|
||||||
writeBoulderState(TEST_DIR, state)
|
writeBoulderState(TEST_DIR, state)
|
||||||
|
|
||||||
const promptMock = mock(() => Promise.reject(new Error("Bad Request")))
|
const promptMock = mock((): Promise<void> => Promise.reject(new Error("Bad Request")))
|
||||||
const mockInput = createMockPluginInput({ promptMock })
|
const mockInput = createMockPluginInput({ promptMock })
|
||||||
const hook = createAtlasHook(mockInput)
|
const hook = createAtlasHook(mockInput)
|
||||||
|
|
||||||
@@ -1769,25 +1769,23 @@ session_id: ses_untrusted_999
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
//#when - idle fires repeatedly, past cooldown each time
|
//#when - idle fires repeatedly, past cooldown each time
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
for (let i = 0; i < 10; i++) {
|
||||||
await flushMicrotasks()
|
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
||||||
now += 6000
|
await flushMicrotasks()
|
||||||
|
now += 6000
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
}
|
||||||
await flushMicrotasks()
|
|
||||||
now += 6000
|
|
||||||
|
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
||||||
await flushMicrotasks()
|
await flushMicrotasks()
|
||||||
|
|
||||||
//#then - should attempt only twice, then disable continuation
|
//#then - should attempt only 10 times, then disable continuation
|
||||||
expect(promptMock).toHaveBeenCalledTimes(2)
|
expect(promptMock).toHaveBeenCalledTimes(10)
|
||||||
} finally {
|
} finally {
|
||||||
Date.now = originalDateNow
|
Date.now = originalDateNow
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
test("should reset prompt failure counter on success and only stop after 2 consecutive failures", async () => {
|
test("should reset prompt failure counter on success and only stop after 10 consecutive failures", async () => {
|
||||||
//#given - boulder state with incomplete plan
|
//#given - boulder state with incomplete plan
|
||||||
const planPath = join(TEST_DIR, "test-plan.md")
|
const planPath = join(TEST_DIR, "test-plan.md")
|
||||||
writeFileSync(planPath, "# Plan\n- [ ] Task 1\n- [ ] Task 2")
|
writeFileSync(planPath, "# Plan\n- [ ] Task 1\n- [ ] Task 2")
|
||||||
@@ -1800,11 +1798,9 @@ session_id: ses_untrusted_999
|
|||||||
}
|
}
|
||||||
writeBoulderState(TEST_DIR, state)
|
writeBoulderState(TEST_DIR, state)
|
||||||
|
|
||||||
const promptMock = mock(() => Promise.resolve())
|
const promptMock = mock((): Promise<void> => Promise.reject(new Error("Bad Request")))
|
||||||
promptMock.mockImplementationOnce(() => Promise.reject(new Error("Bad Request")))
|
promptMock.mockImplementationOnce(() => Promise.reject(new Error("Bad Request")))
|
||||||
promptMock.mockImplementationOnce(() => Promise.resolve())
|
promptMock.mockImplementationOnce(() => Promise.resolve())
|
||||||
promptMock.mockImplementationOnce(() => Promise.reject(new Error("Bad Request")))
|
|
||||||
promptMock.mockImplementationOnce(() => Promise.reject(new Error("Bad Request")))
|
|
||||||
|
|
||||||
const mockInput = createMockPluginInput({ promptMock })
|
const mockInput = createMockPluginInput({ promptMock })
|
||||||
const hook = createAtlasHook(mockInput)
|
const hook = createAtlasHook(mockInput)
|
||||||
@@ -1814,21 +1810,21 @@ session_id: ses_untrusted_999
|
|||||||
Date.now = () => now
|
Date.now = () => now
|
||||||
|
|
||||||
try {
|
try {
|
||||||
//#when - fail, succeed (reset), then fail twice (disable), then attempt again
|
//#when - fail, succeed (reset), then fail 10 times (disable), then attempt again
|
||||||
for (let i = 0; i < 5; i++) {
|
for (let i = 0; i < 13; i++) {
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
||||||
await flushMicrotasks()
|
await flushMicrotasks()
|
||||||
now += 6000
|
now += 6000
|
||||||
}
|
}
|
||||||
|
|
||||||
//#then - 4 prompt attempts; 5th idle is skipped after 2 consecutive failures
|
//#then - 12 prompt attempts; 13th idle is skipped after 10 consecutive failures
|
||||||
expect(promptMock).toHaveBeenCalledTimes(4)
|
expect(promptMock).toHaveBeenCalledTimes(12)
|
||||||
} finally {
|
} finally {
|
||||||
Date.now = originalDateNow
|
Date.now = originalDateNow
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
test("should keep skipping continuation during 5-minute backoff after 2 consecutive failures", async () => {
|
test("should keep skipping continuation during 5-minute backoff after 10 consecutive failures", async () => {
|
||||||
//#given - boulder state with incomplete plan and prompt always fails
|
//#given - boulder state with incomplete plan and prompt always fails
|
||||||
const planPath = join(TEST_DIR, "test-plan.md")
|
const planPath = join(TEST_DIR, "test-plan.md")
|
||||||
writeFileSync(planPath, "# Plan\n- [ ] Task 1\n- [ ] Task 2")
|
writeFileSync(planPath, "# Plan\n- [ ] Task 1\n- [ ] Task 2")
|
||||||
@@ -1850,26 +1846,26 @@ session_id: ses_untrusted_999
|
|||||||
Date.now = () => now
|
Date.now = () => now
|
||||||
|
|
||||||
try {
|
try {
|
||||||
//#when - third idle occurs inside 5-minute backoff window
|
//#when - 11th idle occurs inside 5-minute backoff window
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
for (let i = 0; i < 10; i++) {
|
||||||
await flushMicrotasks()
|
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
||||||
now += 6000
|
await flushMicrotasks()
|
||||||
|
now += 6000
|
||||||
|
}
|
||||||
|
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
|
||||||
await flushMicrotasks()
|
|
||||||
now += 60000
|
now += 60000
|
||||||
|
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
||||||
await flushMicrotasks()
|
await flushMicrotasks()
|
||||||
|
|
||||||
//#then - third attempt should still be skipped
|
//#then - 11th attempt should still be skipped
|
||||||
expect(promptMock).toHaveBeenCalledTimes(2)
|
expect(promptMock).toHaveBeenCalledTimes(10)
|
||||||
} finally {
|
} finally {
|
||||||
Date.now = originalDateNow
|
Date.now = originalDateNow
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
test("should retry continuation after 5-minute backoff expires following 2 consecutive failures", async () => {
|
test("should retry continuation after 5-minute backoff expires following 10 consecutive failures", async () => {
|
||||||
//#given - boulder state with incomplete plan and prompt always fails
|
//#given - boulder state with incomplete plan and prompt always fails
|
||||||
const planPath = join(TEST_DIR, "test-plan.md")
|
const planPath = join(TEST_DIR, "test-plan.md")
|
||||||
writeFileSync(planPath, "# Plan\n- [ ] Task 1\n- [ ] Task 2")
|
writeFileSync(planPath, "# Plan\n- [ ] Task 1\n- [ ] Task 2")
|
||||||
@@ -1891,20 +1887,20 @@ session_id: ses_untrusted_999
|
|||||||
Date.now = () => now
|
Date.now = () => now
|
||||||
|
|
||||||
try {
|
try {
|
||||||
//#when - third idle occurs after 5+ minutes
|
//#when - 11th idle occurs after 5+ minutes
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
for (let i = 0; i < 10; i++) {
|
||||||
await flushMicrotasks()
|
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
||||||
now += 6000
|
await flushMicrotasks()
|
||||||
|
now += 6000
|
||||||
|
}
|
||||||
|
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
|
||||||
await flushMicrotasks()
|
|
||||||
now += 300000
|
now += 300000
|
||||||
|
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
||||||
await flushMicrotasks()
|
await flushMicrotasks()
|
||||||
|
|
||||||
//#then - third attempt should run after backoff expiration
|
//#then - 11th attempt should run after backoff expiration
|
||||||
expect(promptMock).toHaveBeenCalledTimes(3)
|
expect(promptMock).toHaveBeenCalledTimes(11)
|
||||||
} finally {
|
} finally {
|
||||||
Date.now = originalDateNow
|
Date.now = originalDateNow
|
||||||
}
|
}
|
||||||
@@ -1924,8 +1920,9 @@ session_id: ses_untrusted_999
|
|||||||
writeBoulderState(TEST_DIR, state)
|
writeBoulderState(TEST_DIR, state)
|
||||||
|
|
||||||
const promptMock = mock((): Promise<void> => Promise.reject(new Error("Bad Request")))
|
const promptMock = mock((): Promise<void> => Promise.reject(new Error("Bad Request")))
|
||||||
promptMock.mockImplementationOnce(() => Promise.reject(new Error("Bad Request")))
|
for (let i = 0; i < 10; i++) {
|
||||||
promptMock.mockImplementationOnce(() => Promise.reject(new Error("Bad Request")))
|
promptMock.mockImplementationOnce(() => Promise.reject(new Error("Bad Request")))
|
||||||
|
}
|
||||||
promptMock.mockImplementationOnce(() => Promise.resolve(undefined))
|
promptMock.mockImplementationOnce(() => Promise.resolve(undefined))
|
||||||
const mockInput = createMockPluginInput({ promptMock })
|
const mockInput = createMockPluginInput({ promptMock })
|
||||||
const hook = createAtlasHook(mockInput)
|
const hook = createAtlasHook(mockInput)
|
||||||
@@ -1935,32 +1932,30 @@ session_id: ses_untrusted_999
|
|||||||
Date.now = () => now
|
Date.now = () => now
|
||||||
|
|
||||||
try {
|
try {
|
||||||
//#when - fail twice, recover after backoff with success, then fail twice again
|
//#when - fail 10 times, recover after backoff with success, then fail 10 times again
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
for (let i = 0; i < 10; i++) {
|
||||||
await flushMicrotasks()
|
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
||||||
now += 6000
|
await flushMicrotasks()
|
||||||
|
now += 6000
|
||||||
|
}
|
||||||
|
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
|
||||||
await flushMicrotasks()
|
|
||||||
now += 300000
|
now += 300000
|
||||||
|
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
||||||
await flushMicrotasks()
|
await flushMicrotasks()
|
||||||
now += 6000
|
now += 6000
|
||||||
|
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
for (let i = 0; i < 10; i++) {
|
||||||
await flushMicrotasks()
|
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
||||||
now += 6000
|
await flushMicrotasks()
|
||||||
|
now += 6000
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
}
|
||||||
await flushMicrotasks()
|
|
||||||
now += 6000
|
|
||||||
|
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
||||||
await flushMicrotasks()
|
await flushMicrotasks()
|
||||||
|
|
||||||
//#then - success retry resets counter, so two additional failures are allowed before skip
|
//#then - success retry resets counter, so 10 additional failures are allowed before skip
|
||||||
expect(promptMock).toHaveBeenCalledTimes(5)
|
expect(promptMock).toHaveBeenCalledTimes(21)
|
||||||
} finally {
|
} finally {
|
||||||
Date.now = originalDateNow
|
Date.now = originalDateNow
|
||||||
}
|
}
|
||||||
@@ -1988,14 +1983,12 @@ session_id: ses_untrusted_999
|
|||||||
Date.now = () => now
|
Date.now = () => now
|
||||||
|
|
||||||
try {
|
try {
|
||||||
//#when - two failures disables continuation, then compaction resets it
|
//#when - 10 failures disable continuation, then compaction resets it
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
for (let i = 0; i < 10; i++) {
|
||||||
await flushMicrotasks()
|
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
||||||
now += 6000
|
await flushMicrotasks()
|
||||||
|
now += 6000
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
}
|
||||||
await flushMicrotasks()
|
|
||||||
now += 6000
|
|
||||||
|
|
||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
||||||
await flushMicrotasks()
|
await flushMicrotasks()
|
||||||
@@ -2006,8 +1999,8 @@ session_id: ses_untrusted_999
|
|||||||
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
await hook.handler({ event: { type: "session.idle", properties: { sessionID: MAIN_SESSION_ID } } })
|
||||||
await flushMicrotasks()
|
await flushMicrotasks()
|
||||||
|
|
||||||
//#then - 2 attempts + 1 after compaction (3 total)
|
//#then - 10 attempts + 1 after compaction (11 total)
|
||||||
expect(promptMock).toHaveBeenCalledTimes(3)
|
expect(promptMock).toHaveBeenCalledTimes(11)
|
||||||
} finally {
|
} finally {
|
||||||
Date.now = originalDateNow
|
Date.now = originalDateNow
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user