fix(oauth+errors): OAuth silent refresh, quota STOP patterns, compaction loop cap
Bug fixes: 1. OAuth token refresh (#3149): buildHttpRequestInit() now attempts silent refresh via refresh_token before triggering full browser re-auth. Added refresh() method to McpOAuthProvider. Includes test isolation fix for discovery mock. 2. Quota error STOP (#3126): Added STOP_MESSAGE_PATTERNS in model-error-classifier that take precedence over RETRYABLE_MESSAGE_PATTERNS. Message-only quota errors now non-retryable. Runtime-fallback: quota_exceeded with 'retrying in' signal still triggers fallback (provider-managed auto-retry). Restored removed patterns. 3. Compaction loop (#3127): MAX_RECOVERY_ATTEMPTS=3 cap + additional suppression guard from opencode session in degradation monitor. Also: refactored extractAutoRetrySignal to auto-retry-signal.ts, new regression tests for quota classifier and compaction degradation monitor.
This commit is contained in:
YeonGyu-Kim
@@ -0,0 +1,32 @@
|
||||
export interface AutoRetrySignal {
|
||||
signal: string
|
||||
}
|
||||
|
||||
const AUTO_RETRY_PATTERNS: Array<(combined: string) => boolean> = [
|
||||
(combined) => /retrying\s+in/i.test(combined),
|
||||
(combined) =>
|
||||
/(?:too\s+many\s+requests|quota\s+will\s+reset\s+after|quota\s*exceeded|usage\s+limit|rate\s+limit|limit\s+reached|all\s+credentials\s+for\s+model|cool(?:ing)?\s*down|exhausted\s+your\s+capacity)/i.test(combined),
|
||||
]
|
||||
|
||||
export function extractAutoRetrySignal(info: Record<string, unknown> | undefined): AutoRetrySignal | undefined {
|
||||
if (!info) return undefined
|
||||
|
||||
const candidates: string[] = []
|
||||
|
||||
const directStatus = info.status
|
||||
if (typeof directStatus === "string") candidates.push(directStatus)
|
||||
|
||||
const summary = info.summary
|
||||
if (typeof summary === "string") candidates.push(summary)
|
||||
|
||||
const message = info.message
|
||||
if (typeof message === "string") candidates.push(message)
|
||||
|
||||
const details = info.details
|
||||
if (typeof details === "string") candidates.push(details)
|
||||
|
||||
const combined = candidates.join("\n")
|
||||
if (!combined) return undefined
|
||||
|
||||
return AUTO_RETRY_PATTERNS.some((test) => test(combined)) ? { signal: combined } : undefined
|
||||
}
|
||||
@@ -11,7 +11,7 @@ import type { RuntimeFallbackConfig } from "../../config"
|
||||
*/
|
||||
export const DEFAULT_CONFIG: Required<RuntimeFallbackConfig> = {
|
||||
enabled: false,
|
||||
retry_on_errors: [402, 429, 500, 502, 503, 504],
|
||||
retry_on_errors: [429, 500, 502, 503, 504],
|
||||
max_fallback_attempts: 3,
|
||||
cooldown_seconds: 60,
|
||||
timeout_seconds: 30,
|
||||
@@ -25,26 +25,21 @@ export const DEFAULT_CONFIG: Required<RuntimeFallbackConfig> = {
|
||||
export const RETRYABLE_ERROR_PATTERNS = [
|
||||
/rate.?limit/i,
|
||||
/too.?many.?requests/i,
|
||||
/quota.?exceeded/i,
|
||||
/quota\s+will\s+reset\s+after/i,
|
||||
/quota.?exceeded/i,
|
||||
/(?:you(?:'ve|\s+have)\s+)?reached\s+your\s+usage\s+limit/i,
|
||||
/all\s+credentials\s+for\s+model/i,
|
||||
/cool(?:ing)?\s+down/i,
|
||||
/exhausted\s+your\s+capacity/i,
|
||||
/usage\s+limit\s+has\s+been\s+reached/i,
|
||||
/all\s+credentials\s+for\s+model/i,
|
||||
/cool(?:ing)?\s+down/i,
|
||||
/model.{0,20}?not.{0,10}?supported/i,
|
||||
/model_not_supported/i,
|
||||
/insufficient.?(?:credits?|funds?|balance)/i,
|
||||
/credit.*balance.*too.*low/i,
|
||||
/service.?unavailable/i,
|
||||
/overloaded/i,
|
||||
/temporarily.?unavailable/i,
|
||||
/try.?again/i,
|
||||
/credit.*balance.*too.*low/i,
|
||||
/insufficient.?(?:credits?|funds?|balance)/i,
|
||||
/subscription.*quota/i,
|
||||
/billing.?(?:hard.?)?limit/i,
|
||||
/payment.?required/i,
|
||||
/out\s+of\s+credits?/i,
|
||||
/(?:^|\s)402(?:\s|$)/,
|
||||
/(?:^|\s)429(?:\s|$)/,
|
||||
/(?:^|\s)503(?:\s|$)/,
|
||||
/(?:^|\s)529(?:\s|$)/,
|
||||
|
||||
@@ -181,113 +181,7 @@ describe("extractStatusCode", () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe("quota error detection (fixes #2747)", () => {
|
||||
test("classifies prettified subscription quota error as quota_exceeded", () => {
|
||||
//#given
|
||||
const error = {
|
||||
name: "AI_APICallError",
|
||||
message: "Subscription quota exceeded. You can continue using free models.",
|
||||
}
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
const retryable = isRetryableError(error, [402, 429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
test("classifies billing hard limit error as quota_exceeded", () => {
|
||||
//#given
|
||||
const error = { message: "You have reached your billing hard limit." }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
})
|
||||
|
||||
test("classifies exhausted capacity error as quota_exceeded", () => {
|
||||
//#given
|
||||
const error = { message: "You have exhausted your capacity on this model." }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
})
|
||||
|
||||
test("classifies out of credits error as quota_exceeded", () => {
|
||||
//#given
|
||||
const error = { message: "Out of credits. Please add more credits to continue." }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
})
|
||||
|
||||
test("treats HTTP 402 Payment Required as retryable", () => {
|
||||
//#given
|
||||
const error = { statusCode: 402, message: "Payment Required" }
|
||||
|
||||
//#when
|
||||
const retryable = isRetryableError(error, [402, 429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
test("matches subscription quota pattern in RETRYABLE_ERROR_PATTERNS", () => {
|
||||
//#given
|
||||
const error = { message: "Subscription quota exceeded. You can continue using free models." }
|
||||
|
||||
//#when
|
||||
const retryable = isRetryableError(error, [429, 503])
|
||||
|
||||
//#then
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
test("treats hard usage-limit wording as retryable", () => {
|
||||
//#given
|
||||
const error = { message: "You've reached your usage limit for this month. Please upgrade to continue." }
|
||||
|
||||
//#when
|
||||
const retryable = isRetryableError(error, [429, 503])
|
||||
|
||||
//#then
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
test("classifies QuotaExceededError by errorName even without quota keywords in message", () => {
|
||||
//#given
|
||||
const error = { name: "QuotaExceededError", message: "Request failed." }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
})
|
||||
|
||||
test("detects payment required errors as retryable", () => {
|
||||
//#given
|
||||
const error = { message: "Error 402: payment required for this request" }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
const retryable = isRetryableError(error, [429, 503])
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
describe("model support fallback", () => {
|
||||
test("detects model_not_supported errors as retryable for fallback chain", () => {
|
||||
//#given
|
||||
const error1 = { message: "model_not_supported" }
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { DEFAULT_CONFIG, RETRYABLE_ERROR_PATTERNS } from "./constants"
|
||||
|
||||
export { extractAutoRetrySignal } from "./auto-retry-signal"
|
||||
|
||||
export function getErrorMessage(error: unknown): string {
|
||||
if (!error) return ""
|
||||
if (typeof error === "string") return error.toLowerCase()
|
||||
@@ -137,44 +139,6 @@ export function classifyErrorType(error: unknown): string | undefined {
|
||||
return undefined
|
||||
}
|
||||
|
||||
export interface AutoRetrySignal {
|
||||
signal: string
|
||||
}
|
||||
|
||||
export const AUTO_RETRY_PATTERNS: Array<(combined: string) => boolean> = [
|
||||
(combined) => /retrying\s+in/i.test(combined),
|
||||
(combined) =>
|
||||
/(?:too\s+many\s+requests|quota\s*exceeded|quota\s+will\s+reset\s+after|usage\s+limit|rate\s+limit|limit\s+reached|all\s+credentials\s+for\s+model|cool(?:ing)?\s*down|exhausted\s+your\s+capacity)/i.test(combined),
|
||||
]
|
||||
|
||||
export function extractAutoRetrySignal(info: Record<string, unknown> | undefined): AutoRetrySignal | undefined {
|
||||
if (!info) return undefined
|
||||
|
||||
const candidates: string[] = []
|
||||
|
||||
const directStatus = info.status
|
||||
if (typeof directStatus === "string") candidates.push(directStatus)
|
||||
|
||||
const summary = info.summary
|
||||
if (typeof summary === "string") candidates.push(summary)
|
||||
|
||||
const message = info.message
|
||||
if (typeof message === "string") candidates.push(message)
|
||||
|
||||
const details = info.details
|
||||
if (typeof details === "string") candidates.push(details)
|
||||
|
||||
const combined = candidates.join("\n")
|
||||
if (!combined) return undefined
|
||||
|
||||
const isAutoRetry = AUTO_RETRY_PATTERNS.some((test) => test(combined))
|
||||
if (isAutoRetry) {
|
||||
return { signal: combined }
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
export function containsErrorContent(
|
||||
parts: Array<{ type?: string; text?: string }> | undefined
|
||||
): { hasError: boolean; errorMessage?: string } {
|
||||
@@ -204,7 +168,10 @@ export function isRetryableError(error: unknown, retryOnErrors: number[]): boole
|
||||
}
|
||||
|
||||
if (errorType === "quota_exceeded") {
|
||||
return true
|
||||
// When a provider signals an auto-retry (e.g. "retrying in ~2 weeks"),
|
||||
// we should still trigger fallback to another model rather than STOP.
|
||||
const hasAutoRetrySignal = /retrying\s+in/i.test(message)
|
||||
return hasAutoRetrySignal
|
||||
}
|
||||
|
||||
if (statusCode && retryOnErrors.includes(statusCode)) {
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
|
||||
import { classifyErrorType, isRetryableError } from "./error-classifier"
|
||||
|
||||
describe("runtime-fallback quota error regressions", () => {
|
||||
test("classifies subscription quota errors as quota_exceeded and stops retry", () => {
|
||||
//#given
|
||||
const error = {
|
||||
name: "AI_APICallError",
|
||||
message: "Subscription quota exceeded. You can continue using free models.",
|
||||
}
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
const retryable = isRetryableError(error, [429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
expect(retryable).toBe(false)
|
||||
})
|
||||
|
||||
test("treats HTTP 402 payment required as non-retryable", () => {
|
||||
//#given
|
||||
const error = { statusCode: 402, message: "Payment Required" }
|
||||
|
||||
//#when
|
||||
const retryable = isRetryableError(error, [429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(retryable).toBe(false)
|
||||
})
|
||||
|
||||
test("keeps HTTP 429 rate limit retryable", () => {
|
||||
//#given
|
||||
const error = { statusCode: 429, message: "Too Many Requests: rate limit reached" }
|
||||
|
||||
//#when
|
||||
const retryable = isRetryableError(error, [429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
test("classifies quota error names as quota_exceeded without retry", () => {
|
||||
//#given
|
||||
const error = { name: "QuotaExceededError", message: "Request failed." }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
const retryable = isRetryableError(error, [429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
expect(retryable).toBe(false)
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user