fix(oauth+errors): OAuth silent refresh, quota STOP patterns, compaction loop cap
Bug fixes: 1. OAuth token refresh (#3149): buildHttpRequestInit() now attempts silent refresh via refresh_token before triggering full browser re-auth. Added refresh() method to McpOAuthProvider. Includes test isolation fix for discovery mock. 2. Quota error STOP (#3126): Added STOP_MESSAGE_PATTERNS in model-error-classifier that take precedence over RETRYABLE_MESSAGE_PATTERNS. Message-only quota errors now non-retryable. Runtime-fallback: quota_exceeded with 'retrying in' signal still triggers fallback (provider-managed auto-retry). Restored removed patterns. 3. Compaction loop (#3127): MAX_RECOVERY_ATTEMPTS=3 cap + additional suppression guard from opencode session in degradation monitor. Also: refactored extractAutoRetrySignal to auto-retry-signal.ts, new regression tests for quota classifier and compaction degradation monitor.
This commit is contained in:
@@ -0,0 +1,127 @@
|
||||
/// <reference types="bun-types" />
|
||||
|
||||
import { afterAll, beforeEach, describe, expect, it, mock } from "bun:test"
|
||||
|
||||
const logMock = mock(() => {})
|
||||
|
||||
mock.module("../shared/logger", () => ({
|
||||
log: logMock,
|
||||
}))
|
||||
|
||||
afterAll(() => { mock.restore() })
|
||||
|
||||
const { createPreemptiveCompactionHook } = await import("./preemptive-compaction")
|
||||
|
||||
type AssistantHistoryMessage = {
|
||||
info: {
|
||||
id: string
|
||||
role: "assistant"
|
||||
}
|
||||
parts: Array<{ type: string; text?: string }>
|
||||
}
|
||||
|
||||
function createMockCtx(sessionHistory: AssistantHistoryMessage[]) {
|
||||
return {
|
||||
client: {
|
||||
session: {
|
||||
messages: mock(() => Promise.resolve({ data: sessionHistory })),
|
||||
summarize: mock(() => Promise.resolve({})),
|
||||
},
|
||||
tui: {
|
||||
showToast: mock(() => Promise.resolve({})),
|
||||
},
|
||||
},
|
||||
directory: "/tmp/test",
|
||||
}
|
||||
}
|
||||
|
||||
function appendAssistantHistory(
|
||||
sessionHistory: AssistantHistoryMessage[],
|
||||
input: {
|
||||
id: string
|
||||
parts: AssistantHistoryMessage["parts"]
|
||||
},
|
||||
): void {
|
||||
sessionHistory.push({
|
||||
info: {
|
||||
id: input.id,
|
||||
role: "assistant",
|
||||
},
|
||||
parts: input.parts,
|
||||
})
|
||||
}
|
||||
|
||||
function buildAssistantUpdate(input: {
|
||||
sessionID: string
|
||||
id: string
|
||||
parts: unknown[]
|
||||
}) {
|
||||
return {
|
||||
event: {
|
||||
type: "message.updated",
|
||||
properties: {
|
||||
info: {
|
||||
id: input.id,
|
||||
role: "assistant",
|
||||
sessionID: input.sessionID,
|
||||
providerID: "opencode",
|
||||
modelID: "kimi-k2.5-free",
|
||||
finish: true,
|
||||
tokens: { input: 1000, output: 10, reasoning: 0, cache: { read: 0, write: 0 } },
|
||||
parts: input.parts,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
describe("preemptive-compaction degradation monitor regressions", () => {
|
||||
beforeEach(() => {
|
||||
logMock.mockClear()
|
||||
})
|
||||
|
||||
it("does not re-arm monitoring after recovery-triggered compaction", async () => {
|
||||
// given
|
||||
const sessionHistory: AssistantHistoryMessage[] = []
|
||||
const ctx = createMockCtx(sessionHistory)
|
||||
const hook = createPreemptiveCompactionHook(ctx as never, {} as never)
|
||||
const sessionID = "ses_recovery_compaction_guard"
|
||||
const stepOnlyParts = [{ type: "step-start" }, { type: "step-finish" }]
|
||||
|
||||
await hook.event({
|
||||
event: {
|
||||
type: "session.compacted",
|
||||
properties: { sessionID },
|
||||
},
|
||||
})
|
||||
|
||||
// when
|
||||
appendAssistantHistory(sessionHistory, { id: "msg_1", parts: stepOnlyParts })
|
||||
await hook.event(buildAssistantUpdate({ sessionID, id: "msg_1", parts: stepOnlyParts }))
|
||||
|
||||
appendAssistantHistory(sessionHistory, { id: "msg_2", parts: stepOnlyParts })
|
||||
await hook.event(buildAssistantUpdate({ sessionID, id: "msg_2", parts: stepOnlyParts }))
|
||||
|
||||
appendAssistantHistory(sessionHistory, { id: "msg_3", parts: stepOnlyParts })
|
||||
await hook.event(buildAssistantUpdate({ sessionID, id: "msg_3", parts: stepOnlyParts }))
|
||||
|
||||
await hook.event({
|
||||
event: {
|
||||
type: "session.compacted",
|
||||
properties: { sessionID },
|
||||
},
|
||||
})
|
||||
|
||||
appendAssistantHistory(sessionHistory, { id: "msg_4", parts: stepOnlyParts })
|
||||
await hook.event(buildAssistantUpdate({ sessionID, id: "msg_4", parts: stepOnlyParts }))
|
||||
|
||||
appendAssistantHistory(sessionHistory, { id: "msg_5", parts: stepOnlyParts })
|
||||
await hook.event(buildAssistantUpdate({ sessionID, id: "msg_5", parts: stepOnlyParts }))
|
||||
|
||||
appendAssistantHistory(sessionHistory, { id: "msg_6", parts: stepOnlyParts })
|
||||
await hook.event(buildAssistantUpdate({ sessionID, id: "msg_6", parts: stepOnlyParts }))
|
||||
|
||||
// then
|
||||
expect(ctx.client.session.summarize).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
@@ -6,6 +6,7 @@ import { resolveCompactionModel } from "./shared/compaction-model-resolver"
|
||||
const PREEMPTIVE_COMPACTION_TIMEOUT_MS = 120_000
|
||||
const POST_COMPACTION_MONITOR_COUNT = 5
|
||||
const POST_COMPACTION_NO_TEXT_THRESHOLD = 3
|
||||
const RECOVERY_COMPACTION_SUPPRESSION_MS = 5_000
|
||||
|
||||
declare function setTimeout(handler: () => void, timeout?: number): unknown
|
||||
declare function clearTimeout(timeoutID: unknown): void
|
||||
@@ -74,6 +75,7 @@ export function createPostCompactionDegradationMonitor(args: {
|
||||
const postCompactionNoTextStreak = new Map<string, number>()
|
||||
const postCompactionRecoveryTriggered = new Set<string>()
|
||||
const postCompactionEpoch = new Map<string, number>()
|
||||
const suppressRecoveryCompactionUntil = new Map<string, number>()
|
||||
const postCompactionRecoveryCount = new Map<string, number>()
|
||||
|
||||
const MAX_RECOVERY_ATTEMPTS = 3
|
||||
@@ -87,6 +89,13 @@ export function createPostCompactionDegradationMonitor(args: {
|
||||
}
|
||||
|
||||
const onSessionCompacted = (sessionID: string): void => {
|
||||
const suppressedUntil = suppressRecoveryCompactionUntil.get(sessionID)
|
||||
if (suppressedUntil && suppressedUntil > Date.now()) {
|
||||
suppressRecoveryCompactionUntil.delete(sessionID)
|
||||
return
|
||||
}
|
||||
suppressRecoveryCompactionUntil.delete(sessionID)
|
||||
|
||||
const nextEpoch = (postCompactionEpoch.get(sessionID) ?? 0) + 1
|
||||
postCompactionEpoch.set(sessionID, nextEpoch)
|
||||
postCompactionRemaining.set(sessionID, POST_COMPACTION_MONITOR_COUNT)
|
||||
@@ -116,6 +125,7 @@ export function createPostCompactionDegradationMonitor(args: {
|
||||
postCompactionRecoveryTriggered.add(sessionID)
|
||||
compactionInProgress.add(sessionID)
|
||||
const recoveryEpoch = postCompactionEpoch.get(sessionID) ?? 0
|
||||
suppressRecoveryCompactionUntil.set(sessionID, Date.now() + RECOVERY_COMPACTION_SUPPRESSION_MS)
|
||||
|
||||
try {
|
||||
const { providerID: targetProviderID, modelID: targetModelID } = resolveCompactionModel(
|
||||
@@ -148,6 +158,7 @@ export function createPostCompactionDegradationMonitor(args: {
|
||||
|
||||
log("[preemptive-compaction] Triggered recovery after post-compaction no-text tail", { sessionID })
|
||||
} catch (error) {
|
||||
suppressRecoveryCompactionUntil.delete(sessionID)
|
||||
log("[preemptive-compaction] Failed to recover post-compaction no-text tail", {
|
||||
sessionID,
|
||||
error: String(error),
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
export interface AutoRetrySignal {
|
||||
signal: string
|
||||
}
|
||||
|
||||
const AUTO_RETRY_PATTERNS: Array<(combined: string) => boolean> = [
|
||||
(combined) => /retrying\s+in/i.test(combined),
|
||||
(combined) =>
|
||||
/(?:too\s+many\s+requests|quota\s+will\s+reset\s+after|quota\s*exceeded|usage\s+limit|rate\s+limit|limit\s+reached|all\s+credentials\s+for\s+model|cool(?:ing)?\s*down|exhausted\s+your\s+capacity)/i.test(combined),
|
||||
]
|
||||
|
||||
export function extractAutoRetrySignal(info: Record<string, unknown> | undefined): AutoRetrySignal | undefined {
|
||||
if (!info) return undefined
|
||||
|
||||
const candidates: string[] = []
|
||||
|
||||
const directStatus = info.status
|
||||
if (typeof directStatus === "string") candidates.push(directStatus)
|
||||
|
||||
const summary = info.summary
|
||||
if (typeof summary === "string") candidates.push(summary)
|
||||
|
||||
const message = info.message
|
||||
if (typeof message === "string") candidates.push(message)
|
||||
|
||||
const details = info.details
|
||||
if (typeof details === "string") candidates.push(details)
|
||||
|
||||
const combined = candidates.join("\n")
|
||||
if (!combined) return undefined
|
||||
|
||||
return AUTO_RETRY_PATTERNS.some((test) => test(combined)) ? { signal: combined } : undefined
|
||||
}
|
||||
@@ -11,7 +11,7 @@ import type { RuntimeFallbackConfig } from "../../config"
|
||||
*/
|
||||
export const DEFAULT_CONFIG: Required<RuntimeFallbackConfig> = {
|
||||
enabled: false,
|
||||
retry_on_errors: [402, 429, 500, 502, 503, 504],
|
||||
retry_on_errors: [429, 500, 502, 503, 504],
|
||||
max_fallback_attempts: 3,
|
||||
cooldown_seconds: 60,
|
||||
timeout_seconds: 30,
|
||||
@@ -25,26 +25,21 @@ export const DEFAULT_CONFIG: Required<RuntimeFallbackConfig> = {
|
||||
export const RETRYABLE_ERROR_PATTERNS = [
|
||||
/rate.?limit/i,
|
||||
/too.?many.?requests/i,
|
||||
/quota.?exceeded/i,
|
||||
/quota\s+will\s+reset\s+after/i,
|
||||
/quota.?exceeded/i,
|
||||
/(?:you(?:'ve|\s+have)\s+)?reached\s+your\s+usage\s+limit/i,
|
||||
/all\s+credentials\s+for\s+model/i,
|
||||
/cool(?:ing)?\s+down/i,
|
||||
/exhausted\s+your\s+capacity/i,
|
||||
/usage\s+limit\s+has\s+been\s+reached/i,
|
||||
/all\s+credentials\s+for\s+model/i,
|
||||
/cool(?:ing)?\s+down/i,
|
||||
/model.{0,20}?not.{0,10}?supported/i,
|
||||
/model_not_supported/i,
|
||||
/insufficient.?(?:credits?|funds?|balance)/i,
|
||||
/credit.*balance.*too.*low/i,
|
||||
/service.?unavailable/i,
|
||||
/overloaded/i,
|
||||
/temporarily.?unavailable/i,
|
||||
/try.?again/i,
|
||||
/credit.*balance.*too.*low/i,
|
||||
/insufficient.?(?:credits?|funds?|balance)/i,
|
||||
/subscription.*quota/i,
|
||||
/billing.?(?:hard.?)?limit/i,
|
||||
/payment.?required/i,
|
||||
/out\s+of\s+credits?/i,
|
||||
/(?:^|\s)402(?:\s|$)/,
|
||||
/(?:^|\s)429(?:\s|$)/,
|
||||
/(?:^|\s)503(?:\s|$)/,
|
||||
/(?:^|\s)529(?:\s|$)/,
|
||||
|
||||
@@ -181,113 +181,7 @@ describe("extractStatusCode", () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe("quota error detection (fixes #2747)", () => {
|
||||
test("classifies prettified subscription quota error as quota_exceeded", () => {
|
||||
//#given
|
||||
const error = {
|
||||
name: "AI_APICallError",
|
||||
message: "Subscription quota exceeded. You can continue using free models.",
|
||||
}
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
const retryable = isRetryableError(error, [402, 429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
test("classifies billing hard limit error as quota_exceeded", () => {
|
||||
//#given
|
||||
const error = { message: "You have reached your billing hard limit." }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
})
|
||||
|
||||
test("classifies exhausted capacity error as quota_exceeded", () => {
|
||||
//#given
|
||||
const error = { message: "You have exhausted your capacity on this model." }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
})
|
||||
|
||||
test("classifies out of credits error as quota_exceeded", () => {
|
||||
//#given
|
||||
const error = { message: "Out of credits. Please add more credits to continue." }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
})
|
||||
|
||||
test("treats HTTP 402 Payment Required as retryable", () => {
|
||||
//#given
|
||||
const error = { statusCode: 402, message: "Payment Required" }
|
||||
|
||||
//#when
|
||||
const retryable = isRetryableError(error, [402, 429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
test("matches subscription quota pattern in RETRYABLE_ERROR_PATTERNS", () => {
|
||||
//#given
|
||||
const error = { message: "Subscription quota exceeded. You can continue using free models." }
|
||||
|
||||
//#when
|
||||
const retryable = isRetryableError(error, [429, 503])
|
||||
|
||||
//#then
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
test("treats hard usage-limit wording as retryable", () => {
|
||||
//#given
|
||||
const error = { message: "You've reached your usage limit for this month. Please upgrade to continue." }
|
||||
|
||||
//#when
|
||||
const retryable = isRetryableError(error, [429, 503])
|
||||
|
||||
//#then
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
test("classifies QuotaExceededError by errorName even without quota keywords in message", () => {
|
||||
//#given
|
||||
const error = { name: "QuotaExceededError", message: "Request failed." }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
})
|
||||
|
||||
test("detects payment required errors as retryable", () => {
|
||||
//#given
|
||||
const error = { message: "Error 402: payment required for this request" }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
const retryable = isRetryableError(error, [429, 503])
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
describe("model support fallback", () => {
|
||||
test("detects model_not_supported errors as retryable for fallback chain", () => {
|
||||
//#given
|
||||
const error1 = { message: "model_not_supported" }
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { DEFAULT_CONFIG, RETRYABLE_ERROR_PATTERNS } from "./constants"
|
||||
|
||||
export { extractAutoRetrySignal } from "./auto-retry-signal"
|
||||
|
||||
export function getErrorMessage(error: unknown): string {
|
||||
if (!error) return ""
|
||||
if (typeof error === "string") return error.toLowerCase()
|
||||
@@ -137,44 +139,6 @@ export function classifyErrorType(error: unknown): string | undefined {
|
||||
return undefined
|
||||
}
|
||||
|
||||
export interface AutoRetrySignal {
|
||||
signal: string
|
||||
}
|
||||
|
||||
export const AUTO_RETRY_PATTERNS: Array<(combined: string) => boolean> = [
|
||||
(combined) => /retrying\s+in/i.test(combined),
|
||||
(combined) =>
|
||||
/(?:too\s+many\s+requests|quota\s*exceeded|quota\s+will\s+reset\s+after|usage\s+limit|rate\s+limit|limit\s+reached|all\s+credentials\s+for\s+model|cool(?:ing)?\s*down|exhausted\s+your\s+capacity)/i.test(combined),
|
||||
]
|
||||
|
||||
export function extractAutoRetrySignal(info: Record<string, unknown> | undefined): AutoRetrySignal | undefined {
|
||||
if (!info) return undefined
|
||||
|
||||
const candidates: string[] = []
|
||||
|
||||
const directStatus = info.status
|
||||
if (typeof directStatus === "string") candidates.push(directStatus)
|
||||
|
||||
const summary = info.summary
|
||||
if (typeof summary === "string") candidates.push(summary)
|
||||
|
||||
const message = info.message
|
||||
if (typeof message === "string") candidates.push(message)
|
||||
|
||||
const details = info.details
|
||||
if (typeof details === "string") candidates.push(details)
|
||||
|
||||
const combined = candidates.join("\n")
|
||||
if (!combined) return undefined
|
||||
|
||||
const isAutoRetry = AUTO_RETRY_PATTERNS.some((test) => test(combined))
|
||||
if (isAutoRetry) {
|
||||
return { signal: combined }
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
export function containsErrorContent(
|
||||
parts: Array<{ type?: string; text?: string }> | undefined
|
||||
): { hasError: boolean; errorMessage?: string } {
|
||||
@@ -204,7 +168,10 @@ export function isRetryableError(error: unknown, retryOnErrors: number[]): boole
|
||||
}
|
||||
|
||||
if (errorType === "quota_exceeded") {
|
||||
return true
|
||||
// When a provider signals an auto-retry (e.g. "retrying in ~2 weeks"),
|
||||
// we should still trigger fallback to another model rather than STOP.
|
||||
const hasAutoRetrySignal = /retrying\s+in/i.test(message)
|
||||
return hasAutoRetrySignal
|
||||
}
|
||||
|
||||
if (statusCode && retryOnErrors.includes(statusCode)) {
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
|
||||
import { classifyErrorType, isRetryableError } from "./error-classifier"
|
||||
|
||||
describe("runtime-fallback quota error regressions", () => {
|
||||
test("classifies subscription quota errors as quota_exceeded and stops retry", () => {
|
||||
//#given
|
||||
const error = {
|
||||
name: "AI_APICallError",
|
||||
message: "Subscription quota exceeded. You can continue using free models.",
|
||||
}
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
const retryable = isRetryableError(error, [429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
expect(retryable).toBe(false)
|
||||
})
|
||||
|
||||
test("treats HTTP 402 payment required as non-retryable", () => {
|
||||
//#given
|
||||
const error = { statusCode: 402, message: "Payment Required" }
|
||||
|
||||
//#when
|
||||
const retryable = isRetryableError(error, [429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(retryable).toBe(false)
|
||||
})
|
||||
|
||||
test("keeps HTTP 429 rate limit retryable", () => {
|
||||
//#given
|
||||
const error = { statusCode: 429, message: "Too Many Requests: rate limit reached" }
|
||||
|
||||
//#when
|
||||
const retryable = isRetryableError(error, [429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
test("classifies quota error names as quota_exceeded without retry", () => {
|
||||
//#given
|
||||
const error = { name: "QuotaExceededError", message: "Request failed." }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
const retryable = isRetryableError(error, [429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
expect(retryable).toBe(false)
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user