refactor(circuit-breaker): replace sliding window with consecutive call detection

Switch background task loop detection from percentage-based sliding window
(80% of 20-call window) to consecutive same-tool counting. Triggers when
same tool signature is called 20+ times in a row; a different tool resets
the counter.
This commit is contained in:
YeonGyu-Kim
2026-03-18 14:32:27 +09:00
parent c5c7ba4eed
commit d48ea025f0
8 changed files with 98 additions and 187 deletions
@@ -8,27 +8,24 @@ describe("BackgroundTaskConfigSchema.circuitBreaker", () => {
const result = BackgroundTaskConfigSchema.parse({ const result = BackgroundTaskConfigSchema.parse({
circuitBreaker: { circuitBreaker: {
maxToolCalls: 150, maxToolCalls: 150,
windowSize: 10, consecutiveThreshold: 10,
repetitionThresholdPercent: 70,
}, },
}) })
expect(result.circuitBreaker).toEqual({ expect(result.circuitBreaker).toEqual({
maxToolCalls: 150, maxToolCalls: 150,
windowSize: 10, consecutiveThreshold: 10,
repetitionThresholdPercent: 70,
}) })
}) })
}) })
describe("#given windowSize below minimum", () => { describe("#given consecutiveThreshold below minimum", () => {
test("#when parsed #then throws ZodError", () => { test("#when parsed #then throws ZodError", () => {
let thrownError: unknown let thrownError: unknown
try { try {
BackgroundTaskConfigSchema.parse({ BackgroundTaskConfigSchema.parse({
circuitBreaker: { circuitBreaker: {
windowSize: 4, consecutiveThreshold: 4,
}, },
}) })
} catch (error) { } catch (error) {
@@ -39,14 +36,14 @@ describe("BackgroundTaskConfigSchema.circuitBreaker", () => {
}) })
}) })
describe("#given repetitionThresholdPercent is zero", () => { describe("#given consecutiveThreshold is zero", () => {
test("#when parsed #then throws ZodError", () => { test("#when parsed #then throws ZodError", () => {
let thrownError: unknown let thrownError: unknown
try { try {
BackgroundTaskConfigSchema.parse({ BackgroundTaskConfigSchema.parse({
circuitBreaker: { circuitBreaker: {
repetitionThresholdPercent: 0, consecutiveThreshold: 0,
}, },
}) })
} catch (error) { } catch (error) {
+1 -2
View File
@@ -3,8 +3,7 @@ import { z } from "zod"
const CircuitBreakerConfigSchema = z.object({ const CircuitBreakerConfigSchema = z.object({
enabled: z.boolean().optional(), enabled: z.boolean().optional(),
maxToolCalls: z.number().int().min(10).optional(), maxToolCalls: z.number().int().min(10).optional(),
windowSize: z.number().int().min(5).optional(), consecutiveThreshold: z.number().int().min(5).optional(),
repetitionThresholdPercent: z.number().gt(0).max(100).optional(),
}) })
export const BackgroundTaskConfigSchema = z.object({ export const BackgroundTaskConfigSchema = z.object({
+1 -2
View File
@@ -7,8 +7,7 @@ export const MIN_STABILITY_TIME_MS = 10 * 1000
export const DEFAULT_STALE_TIMEOUT_MS = 1_200_000 export const DEFAULT_STALE_TIMEOUT_MS = 1_200_000
export const DEFAULT_MESSAGE_STALENESS_TIMEOUT_MS = 1_800_000 export const DEFAULT_MESSAGE_STALENESS_TIMEOUT_MS = 1_800_000
export const DEFAULT_MAX_TOOL_CALLS = 200 export const DEFAULT_MAX_TOOL_CALLS = 200
export const DEFAULT_CIRCUIT_BREAKER_WINDOW_SIZE = 20 export const DEFAULT_CIRCUIT_BREAKER_CONSECUTIVE_THRESHOLD = 20
export const DEFAULT_CIRCUIT_BREAKER_REPETITION_THRESHOLD_PERCENT = 80
export const DEFAULT_CIRCUIT_BREAKER_ENABLED = true export const DEFAULT_CIRCUIT_BREAKER_ENABLED = true
export const MIN_RUNTIME_BEFORE_STALE_MS = 30_000 export const MIN_RUNTIME_BEFORE_STALE_MS = 30_000
export const MIN_IDLE_TIME_MS = 5000 export const MIN_IDLE_TIME_MS = 5000
@@ -37,16 +37,14 @@ describe("loop-detector", () => {
maxToolCalls: 200, maxToolCalls: 200,
circuitBreaker: { circuitBreaker: {
maxToolCalls: 120, maxToolCalls: 120,
windowSize: 10, consecutiveThreshold: 7,
repetitionThresholdPercent: 70,
}, },
}) })
expect(result).toEqual({ expect(result).toEqual({
enabled: true, enabled: true,
maxToolCalls: 120, maxToolCalls: 120,
windowSize: 10, consecutiveThreshold: 7,
repetitionThresholdPercent: 70,
}) })
}) })
}) })
@@ -56,8 +54,7 @@ describe("loop-detector", () => {
const result = resolveCircuitBreakerSettings({ const result = resolveCircuitBreakerSettings({
circuitBreaker: { circuitBreaker: {
maxToolCalls: 100, maxToolCalls: 100,
windowSize: 5, consecutiveThreshold: 5,
repetitionThresholdPercent: 60,
}, },
}) })
@@ -71,8 +68,7 @@ describe("loop-detector", () => {
circuitBreaker: { circuitBreaker: {
enabled: false, enabled: false,
maxToolCalls: 100, maxToolCalls: 100,
windowSize: 5, consecutiveThreshold: 5,
repetitionThresholdPercent: 60,
}, },
}) })
@@ -86,8 +82,7 @@ describe("loop-detector", () => {
circuitBreaker: { circuitBreaker: {
enabled: true, enabled: true,
maxToolCalls: 100, maxToolCalls: 100,
windowSize: 5, consecutiveThreshold: 5,
repetitionThresholdPercent: 60,
}, },
}) })
@@ -151,55 +146,52 @@ describe("loop-detector", () => {
}) })
}) })
describe("#given the same tool dominates the recent window", () => { describe("#given the same tool is called consecutively", () => {
test("#when evaluated #then it triggers", () => { test("#when evaluated #then it triggers", () => {
const window = buildWindow([ const window = buildWindow(Array.from({ length: 20 }, () => "read"))
"read",
"read",
"read",
"edit",
"read",
"read",
"read",
"read",
"grep",
"read",
], {
circuitBreaker: {
windowSize: 10,
repetitionThresholdPercent: 80,
},
})
const result = detectRepetitiveToolUse(window) const result = detectRepetitiveToolUse(window)
expect(result).toEqual({ expect(result).toEqual({
triggered: true, triggered: true,
toolName: "read", toolName: "read",
repeatedCount: 8, repeatedCount: 20,
sampleSize: 10,
thresholdPercent: 80,
}) })
}) })
}) })
describe("#given the window is not full yet", () => { describe("#given consecutive calls are interrupted by different tool", () => {
test("#when the current sample crosses the threshold #then it still triggers", () => { test("#when evaluated #then it does not trigger", () => {
const window = buildWindow(["read", "read", "edit", "read", "read", "read", "read", "read"], { const window = buildWindow([
circuitBreaker: { ...Array.from({ length: 19 }, () => "read"),
windowSize: 10, "edit",
repetitionThresholdPercent: 80, "read",
}, ])
})
const result = detectRepetitiveToolUse(window) const result = detectRepetitiveToolUse(window)
expect(result).toEqual({ triggered: false })
})
})
describe("#given threshold boundary", () => {
test("#when below threshold #then it does not trigger", () => {
const belowThresholdWindow = buildWindow(Array.from({ length: 19 }, () => "read"))
const result = detectRepetitiveToolUse(belowThresholdWindow)
expect(result).toEqual({ triggered: false })
})
test("#when equal to threshold #then it triggers", () => {
const atThresholdWindow = buildWindow(Array.from({ length: 20 }, () => "read"))
const result = detectRepetitiveToolUse(atThresholdWindow)
expect(result).toEqual({ expect(result).toEqual({
triggered: true, triggered: true,
toolName: "read", toolName: "read",
repeatedCount: 7, repeatedCount: 20,
sampleSize: 8,
thresholdPercent: 80,
}) })
}) })
}) })
@@ -210,9 +202,7 @@ describe("loop-detector", () => {
tool: "read", tool: "read",
input: { filePath: `/src/file-${i}.ts` }, input: { filePath: `/src/file-${i}.ts` },
})) }))
const window = buildWindowWithInputs(calls, { const window = buildWindowWithInputs(calls)
circuitBreaker: { windowSize: 20, repetitionThresholdPercent: 80 },
})
const result = detectRepetitiveToolUse(window) const result = detectRepetitiveToolUse(window)
expect(result.triggered).toBe(false) expect(result.triggered).toBe(false)
}) })
@@ -220,38 +210,30 @@ describe("loop-detector", () => {
describe("#given same tool with identical file inputs", () => { describe("#given same tool with identical file inputs", () => {
test("#when evaluated #then it triggers with bare tool name", () => { test("#when evaluated #then it triggers with bare tool name", () => {
const calls = [ const calls = Array.from({ length: 20 }, () => ({
...Array.from({ length: 16 }, () => ({ tool: "read", input: { filePath: "/src/same.ts" } })), tool: "read",
{ tool: "grep", input: { pattern: "foo" } }, input: { filePath: "/src/same.ts" },
{ tool: "edit", input: { filePath: "/src/other.ts" } }, }))
{ tool: "bash", input: { command: "ls" } }, const window = buildWindowWithInputs(calls)
{ tool: "glob", input: { pattern: "**/*.ts" } },
]
const window = buildWindowWithInputs(calls, {
circuitBreaker: { windowSize: 20, repetitionThresholdPercent: 80 },
})
const result = detectRepetitiveToolUse(window) const result = detectRepetitiveToolUse(window)
expect(result.triggered).toBe(true) expect(result).toEqual({
expect(result.toolName).toBe("read") triggered: true,
expect(result.repeatedCount).toBe(16) toolName: "read",
repeatedCount: 20,
})
}) })
}) })
describe("#given tool calls with no input", () => { describe("#given tool calls with no input", () => {
test("#when the same tool dominates #then falls back to name-only detection", () => { test("#when evaluated #then it triggers", () => {
const calls = [ const calls = Array.from({ length: 20 }, () => ({ tool: "read" }))
...Array.from({ length: 16 }, () => ({ tool: "read" })), const window = buildWindowWithInputs(calls)
{ tool: "grep" },
{ tool: "edit" },
{ tool: "bash" },
{ tool: "glob" },
]
const window = buildWindowWithInputs(calls, {
circuitBreaker: { windowSize: 20, repetitionThresholdPercent: 80 },
})
const result = detectRepetitiveToolUse(window) const result = detectRepetitiveToolUse(window)
expect(result.triggered).toBe(true) expect(result).toEqual({
expect(result.toolName).toBe("read") triggered: true,
toolName: "read",
repeatedCount: 20,
})
}) })
}) })
}) })
+18 -53
View File
@@ -1,8 +1,7 @@
import type { BackgroundTaskConfig } from "../../config/schema" import type { BackgroundTaskConfig } from "../../config/schema"
import { import {
DEFAULT_CIRCUIT_BREAKER_ENABLED, DEFAULT_CIRCUIT_BREAKER_ENABLED,
DEFAULT_CIRCUIT_BREAKER_REPETITION_THRESHOLD_PERCENT, DEFAULT_CIRCUIT_BREAKER_CONSECUTIVE_THRESHOLD,
DEFAULT_CIRCUIT_BREAKER_WINDOW_SIZE,
DEFAULT_MAX_TOOL_CALLS, DEFAULT_MAX_TOOL_CALLS,
} from "./constants" } from "./constants"
import type { ToolCallWindow } from "./types" import type { ToolCallWindow } from "./types"
@@ -10,16 +9,13 @@ import type { ToolCallWindow } from "./types"
export interface CircuitBreakerSettings { export interface CircuitBreakerSettings {
enabled: boolean enabled: boolean
maxToolCalls: number maxToolCalls: number
windowSize: number consecutiveThreshold: number
repetitionThresholdPercent: number
} }
export interface ToolLoopDetectionResult { export interface ToolLoopDetectionResult {
triggered: boolean triggered: boolean
toolName?: string toolName?: string
repeatedCount?: number repeatedCount?: number
sampleSize?: number
thresholdPercent?: number
} }
export function resolveCircuitBreakerSettings( export function resolveCircuitBreakerSettings(
@@ -29,10 +25,8 @@ export function resolveCircuitBreakerSettings(
enabled: config?.circuitBreaker?.enabled ?? DEFAULT_CIRCUIT_BREAKER_ENABLED, enabled: config?.circuitBreaker?.enabled ?? DEFAULT_CIRCUIT_BREAKER_ENABLED,
maxToolCalls: maxToolCalls:
config?.circuitBreaker?.maxToolCalls ?? config?.maxToolCalls ?? DEFAULT_MAX_TOOL_CALLS, config?.circuitBreaker?.maxToolCalls ?? config?.maxToolCalls ?? DEFAULT_MAX_TOOL_CALLS,
windowSize: config?.circuitBreaker?.windowSize ?? DEFAULT_CIRCUIT_BREAKER_WINDOW_SIZE, consecutiveThreshold:
repetitionThresholdPercent: config?.circuitBreaker?.consecutiveThreshold ?? DEFAULT_CIRCUIT_BREAKER_CONSECUTIVE_THRESHOLD,
config?.circuitBreaker?.repetitionThresholdPercent ??
DEFAULT_CIRCUIT_BREAKER_REPETITION_THRESHOLD_PERCENT,
} }
} }
@@ -42,16 +36,20 @@ export function recordToolCall(
settings: CircuitBreakerSettings, settings: CircuitBreakerSettings,
toolInput?: Record<string, unknown> | null toolInput?: Record<string, unknown> | null
): ToolCallWindow { ): ToolCallWindow {
const previous = window?.toolSignatures ?? []
const signature = createToolCallSignature(toolName, toolInput) const signature = createToolCallSignature(toolName, toolInput)
const toolSignatures = previous.length >= settings.windowSize
? [...previous.slice(1), signature] if (window && window.lastSignature === signature) {
: [...previous, signature] return {
lastSignature: signature,
consecutiveCount: window.consecutiveCount + 1,
threshold: settings.consecutiveThreshold,
}
}
return { return {
toolSignatures, lastSignature: signature,
windowSize: settings.windowSize, consecutiveCount: 1,
thresholdPercent: settings.repetitionThresholdPercent, threshold: settings.consecutiveThreshold,
} }
} }
@@ -84,46 +82,13 @@ export function createToolCallSignature(
export function detectRepetitiveToolUse( export function detectRepetitiveToolUse(
window: ToolCallWindow | undefined window: ToolCallWindow | undefined
): ToolLoopDetectionResult { ): ToolLoopDetectionResult {
if (!window || window.toolSignatures.length === 0) { if (!window || window.consecutiveCount < window.threshold) {
return { triggered: false }
}
const counts = new Map<string, number>()
for (const signature of window.toolSignatures) {
counts.set(signature, (counts.get(signature) ?? 0) + 1)
}
let repeatedTool: string | undefined
let repeatedCount = 0
for (const [toolName, count] of counts.entries()) {
if (count > repeatedCount) {
repeatedTool = toolName
repeatedCount = count
}
}
const sampleSize = window.toolSignatures.length
const minimumSampleSize = Math.min(
window.windowSize,
Math.ceil((window.windowSize * window.thresholdPercent) / 100)
)
if (sampleSize < minimumSampleSize) {
return { triggered: false }
}
const thresholdCount = Math.ceil((sampleSize * window.thresholdPercent) / 100)
if (!repeatedTool || repeatedCount < thresholdCount) {
return { triggered: false } return { triggered: false }
} }
return { return {
triggered: true, triggered: true,
toolName: repeatedTool.split("::")[0], toolName: window.lastSignature.split("::")[0],
repeatedCount, repeatedCount: window.consecutiveCount,
sampleSize,
thresholdPercent: window.thresholdPercent,
} }
} }
@@ -38,12 +38,11 @@ async function flushAsyncWork() {
} }
describe("BackgroundManager circuit breaker", () => { describe("BackgroundManager circuit breaker", () => {
describe("#given the same tool dominates the recent window", () => { describe("#given the same tool is called consecutively", () => {
test("#when tool events arrive #then the task is cancelled early", async () => { test("#when consecutive tool events arrive #then the task is cancelled", async () => {
const manager = createManager({ const manager = createManager({
circuitBreaker: { circuitBreaker: {
windowSize: 20, consecutiveThreshold: 20,
repetitionThresholdPercent: 80,
}, },
}) })
const task: BackgroundTask = { const task: BackgroundTask = {
@@ -63,38 +62,17 @@ describe("BackgroundManager circuit breaker", () => {
} }
getTaskMap(manager).set(task.id, task) getTaskMap(manager).set(task.id, task)
for (const toolName of [ for (let i = 0; i < 20; i++) {
"read",
"read",
"grep",
"read",
"edit",
"read",
"read",
"bash",
"read",
"read",
"read",
"glob",
"read",
"read",
"read",
"read",
"read",
"read",
"read",
"read",
]) {
manager.handleEvent({ manager.handleEvent({
type: "message.part.updated", type: "message.part.updated",
properties: { sessionID: task.sessionID, type: "tool", tool: toolName }, properties: { sessionID: task.sessionID, type: "tool", tool: "read" },
}) })
} }
await flushAsyncWork() await flushAsyncWork()
expect(task.status).toBe("cancelled") expect(task.status).toBe("cancelled")
expect(task.error).toContain("repeatedly called read 16/20 times") expect(task.error).toContain("read 20 consecutive times")
}) })
}) })
@@ -102,8 +80,7 @@ describe("BackgroundManager circuit breaker", () => {
test("#when the window fills #then the task keeps running", async () => { test("#when the window fills #then the task keeps running", async () => {
const manager = createManager({ const manager = createManager({
circuitBreaker: { circuitBreaker: {
windowSize: 10, consecutiveThreshold: 10,
repetitionThresholdPercent: 80,
}, },
}) })
const task: BackgroundTask = { const task: BackgroundTask = {
@@ -153,8 +130,7 @@ describe("BackgroundManager circuit breaker", () => {
const manager = createManager({ const manager = createManager({
maxToolCalls: 3, maxToolCalls: 3,
circuitBreaker: { circuitBreaker: {
windowSize: 10, consecutiveThreshold: 95,
repetitionThresholdPercent: 95,
}, },
}) })
const task: BackgroundTask = { const task: BackgroundTask = {
@@ -193,8 +169,7 @@ describe("BackgroundManager circuit breaker", () => {
const manager = createManager({ const manager = createManager({
maxToolCalls: 2, maxToolCalls: 2,
circuitBreaker: { circuitBreaker: {
windowSize: 5, consecutiveThreshold: 5,
repetitionThresholdPercent: 80,
}, },
}) })
const task: BackgroundTask = { const task: BackgroundTask = {
@@ -241,8 +216,7 @@ describe("BackgroundManager circuit breaker", () => {
test("#when tool events arrive with state.input #then task keeps running", async () => { test("#when tool events arrive with state.input #then task keeps running", async () => {
const manager = createManager({ const manager = createManager({
circuitBreaker: { circuitBreaker: {
windowSize: 20, consecutiveThreshold: 20,
repetitionThresholdPercent: 80,
}, },
}) })
const task: BackgroundTask = { const task: BackgroundTask = {
@@ -287,8 +261,7 @@ describe("BackgroundManager circuit breaker", () => {
test("#when tool events arrive with state.input #then task is cancelled with bare tool name in error", async () => { test("#when tool events arrive with state.input #then task is cancelled with bare tool name in error", async () => {
const manager = createManager({ const manager = createManager({
circuitBreaker: { circuitBreaker: {
windowSize: 20, consecutiveThreshold: 20,
repetitionThresholdPercent: 80,
}, },
}) })
const task: BackgroundTask = { const task: BackgroundTask = {
@@ -325,7 +298,7 @@ describe("BackgroundManager circuit breaker", () => {
await flushAsyncWork() await flushAsyncWork()
expect(task.status).toBe("cancelled") expect(task.status).toBe("cancelled")
expect(task.error).toContain("repeatedly called read") expect(task.error).toContain("read 20 consecutive times")
expect(task.error).not.toContain("::") expect(task.error).not.toContain("::")
}) })
}) })
@@ -335,8 +308,7 @@ describe("BackgroundManager circuit breaker", () => {
const manager = createManager({ const manager = createManager({
circuitBreaker: { circuitBreaker: {
enabled: false, enabled: false,
windowSize: 20, consecutiveThreshold: 20,
repetitionThresholdPercent: 80,
}, },
}) })
const task: BackgroundTask = { const task: BackgroundTask = {
@@ -379,8 +351,7 @@ describe("BackgroundManager circuit breaker", () => {
maxToolCalls: 3, maxToolCalls: 3,
circuitBreaker: { circuitBreaker: {
enabled: false, enabled: false,
windowSize: 10, consecutiveThreshold: 95,
repetitionThresholdPercent: 95,
}, },
}) })
const task: BackgroundTask = { const task: BackgroundTask = {
+2 -4
View File
@@ -932,18 +932,16 @@ export class BackgroundManager {
if (circuitBreaker.enabled) { if (circuitBreaker.enabled) {
const loopDetection = detectRepetitiveToolUse(task.progress.toolCallWindow) const loopDetection = detectRepetitiveToolUse(task.progress.toolCallWindow)
if (loopDetection.triggered) { if (loopDetection.triggered) {
log("[background-agent] Circuit breaker: repetitive tool usage detected", { log("[background-agent] Circuit breaker: consecutive tool usage detected", {
taskId: task.id, taskId: task.id,
agent: task.agent, agent: task.agent,
sessionID, sessionID,
toolName: loopDetection.toolName, toolName: loopDetection.toolName,
repeatedCount: loopDetection.repeatedCount, repeatedCount: loopDetection.repeatedCount,
sampleSize: loopDetection.sampleSize,
thresholdPercent: loopDetection.thresholdPercent,
}) })
void this.cancelTask(task.id, { void this.cancelTask(task.id, {
source: "circuit-breaker", source: "circuit-breaker",
reason: `Subagent repeatedly called ${loopDetection.toolName} ${loopDetection.repeatedCount}/${loopDetection.sampleSize} times in the recent tool-call window (${loopDetection.thresholdPercent}% threshold). This usually indicates an infinite loop. The task was automatically cancelled to prevent excessive token usage.`, reason: `Subagent called ${loopDetection.toolName} ${loopDetection.repeatedCount} consecutive times (threshold: ${circuitBreaker.consecutiveThreshold}). This usually indicates an infinite loop. The task was automatically cancelled to prevent excessive token usage.`,
}) })
return return
} }
+3 -3
View File
@@ -10,9 +10,9 @@ export type BackgroundTaskStatus =
| "interrupt" | "interrupt"
export interface ToolCallWindow { export interface ToolCallWindow {
toolSignatures: string[] lastSignature: string
windowSize: number consecutiveCount: number
thresholdPercent: number threshold: number
} }
export interface TaskProgress { export interface TaskProgress {