fix(oauth+errors): OAuth silent refresh, quota STOP patterns, compaction loop cap
Bug fixes: 1. OAuth token refresh (#3149): buildHttpRequestInit() now attempts silent refresh via refresh_token before triggering full browser re-auth. Added refresh() method to McpOAuthProvider. Includes test isolation fix for discovery mock. 2. Quota error STOP (#3126): Added STOP_MESSAGE_PATTERNS in model-error-classifier that take precedence over RETRYABLE_MESSAGE_PATTERNS. Message-only quota errors now non-retryable. Runtime-fallback: quota_exceeded with 'retrying in' signal still triggers fallback (provider-managed auto-retry). Restored removed patterns. 3. Compaction loop (#3127): MAX_RECOVERY_ATTEMPTS=3 cap + additional suppression guard from opencode session in degradation monitor. Also: refactored extractAutoRetrySignal to auto-retry-signal.ts, new regression tests for quota classifier and compaction degradation monitor.
This commit is contained in:
+101
-336
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, it, beforeEach, afterEach, mock } from "bun:test"
|
||||
import { describe, expect, it, beforeEach, afterEach, mock, afterAll } from "bun:test"
|
||||
import { createHash, randomBytes } from "node:crypto"
|
||||
import type { OAuthTokenData } from "./storage"
|
||||
|
||||
@@ -226,6 +226,90 @@ describe("McpOAuthProvider", () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe("refresh", () => {
|
||||
let originalFetch: typeof globalThis.fetch
|
||||
let originalEnv: string | undefined
|
||||
|
||||
beforeEach(() => {
|
||||
originalFetch = globalThis.fetch
|
||||
originalEnv = process.env.OPENCODE_CONFIG_DIR
|
||||
const { mkdirSync } = require("node:fs")
|
||||
const { tmpdir } = require("node:os")
|
||||
const { join } = require("node:path")
|
||||
const testDir = join(tmpdir(), `mcp-oauth-provider-refresh-test-${Date.now()}`)
|
||||
mkdirSync(testDir, { recursive: true })
|
||||
process.env.OPENCODE_CONFIG_DIR = testDir
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch
|
||||
if (originalEnv === undefined) {
|
||||
delete process.env.OPENCODE_CONFIG_DIR
|
||||
} else {
|
||||
process.env.OPENCODE_CONFIG_DIR = originalEnv
|
||||
}
|
||||
})
|
||||
|
||||
it("exchanges refresh token and preserves it when the response omits a new one", async () => {
|
||||
// Stub fetch to handle both discovery (well-known) and token exchange
|
||||
const fetchStub = mock(async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const url = input.toString()
|
||||
if (url.includes("oauth-protected-resource")) {
|
||||
// PRM: return authorization_servers pointing to auth server
|
||||
return new Response(
|
||||
JSON.stringify({ authorization_servers: ["https://auth.example.com"] }),
|
||||
{ status: 200, headers: { "content-type": "application/json" } },
|
||||
)
|
||||
}
|
||||
if (url.includes(".well-known")) {
|
||||
// AS metadata
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
issuer: "https://auth.example.com",
|
||||
authorization_endpoint: "https://auth.example.com/authorize",
|
||||
token_endpoint: "https://auth.example.com/token",
|
||||
}),
|
||||
{ status: 200, headers: { "content-type": "application/json" } },
|
||||
)
|
||||
}
|
||||
// Token exchange
|
||||
const body = init?.body?.toString() ?? ""
|
||||
expect(body).toContain("grant_type=refresh_token")
|
||||
expect(body).toContain("refresh_token=refresh-token-456")
|
||||
expect(body).toContain("client_id=my-client")
|
||||
return new Response(
|
||||
JSON.stringify({ access_token: "refreshed-access-token", expires_in: 3600 }),
|
||||
{ status: 200, headers: { "content-type": "application/json" } },
|
||||
)
|
||||
})
|
||||
const fetchMock = Object.assign(
|
||||
async (...args: Parameters<typeof fetch>): ReturnType<typeof fetch> => fetchStub(...args),
|
||||
{ preconnect: originalFetch.preconnect.bind(originalFetch) },
|
||||
) satisfies typeof fetch
|
||||
globalThis.fetch = fetchMock
|
||||
|
||||
// given
|
||||
const providerModule = await importFreshProviderModule()
|
||||
const provider = new providerModule.McpOAuthProvider({
|
||||
serverUrl: "https://mcp.example.com",
|
||||
clientId: "my-client",
|
||||
})
|
||||
provider.saveTokens({
|
||||
accessToken: "old-access-token",
|
||||
refreshToken: "refresh-token-456",
|
||||
expiresAt: Math.floor(Date.now() / 1000) - 60,
|
||||
clientInfo: { clientId: "my-client" },
|
||||
})
|
||||
|
||||
// when
|
||||
const result = await provider.refresh("refresh-token-456")
|
||||
|
||||
// then
|
||||
expect(result.accessToken).toBe("refreshed-access-token")
|
||||
expect(result.refreshToken).toBe("refresh-token-456") // preserved from input when absent in response
|
||||
})
|
||||
})
|
||||
|
||||
describe("redirectUrl", () => {
|
||||
it("returns localhost callback URL with default port", () => {
|
||||
// given
|
||||
|
||||
@@ -19,6 +19,48 @@ export type McpOAuthProviderOptions = {
|
||||
scopes?: string[]
|
||||
}
|
||||
|
||||
async function parseTokenResponse(tokenResponse: Response): Promise<Record<string, unknown>> {
|
||||
if (!tokenResponse.ok) {
|
||||
let errorDetail = `${tokenResponse.status}`
|
||||
try {
|
||||
const body = (await tokenResponse.json()) as Record<string, unknown>
|
||||
if (body.error) {
|
||||
errorDetail = `${tokenResponse.status} ${body.error}`
|
||||
if (body.error_description) {
|
||||
errorDetail += `: ${body.error_description}`
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Response body not JSON
|
||||
}
|
||||
throw new Error(`Token exchange failed: ${errorDetail}`)
|
||||
}
|
||||
|
||||
return (await tokenResponse.json()) as Record<string, unknown>
|
||||
}
|
||||
|
||||
function buildOAuthTokenData(
|
||||
tokenData: Record<string, unknown>,
|
||||
clientInfo: ClientCredentials,
|
||||
fallbackRefreshToken?: string,
|
||||
): OAuthTokenData {
|
||||
const accessToken = tokenData.access_token
|
||||
if (typeof accessToken !== "string") {
|
||||
throw new Error("Token response missing access_token")
|
||||
}
|
||||
|
||||
return {
|
||||
accessToken,
|
||||
refreshToken: typeof tokenData.refresh_token === "string" ? tokenData.refresh_token : fallbackRefreshToken,
|
||||
expiresAt:
|
||||
typeof tokenData.expires_in === "number" ? Math.floor(Date.now() / 1000) + tokenData.expires_in : undefined,
|
||||
clientInfo: {
|
||||
clientId: clientInfo.clientId,
|
||||
...(clientInfo.clientSecret ? { clientSecret: clientInfo.clientSecret } : {}),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export class McpOAuthProvider {
|
||||
private readonly serverUrl: string
|
||||
private readonly configClientId: string | undefined
|
||||
@@ -131,38 +173,38 @@ export class McpOAuthProvider {
|
||||
}).toString(),
|
||||
})
|
||||
|
||||
if (!tokenResponse.ok) {
|
||||
let errorDetail = `${tokenResponse.status}`
|
||||
try {
|
||||
const body = (await tokenResponse.json()) as Record<string, unknown>
|
||||
if (body.error) {
|
||||
errorDetail = `${tokenResponse.status} ${body.error}`
|
||||
if (body.error_description) {
|
||||
errorDetail += `: ${body.error_description}`
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Response body not JSON
|
||||
}
|
||||
throw new Error(`Token exchange failed: ${errorDetail}`)
|
||||
const tokenData = await parseTokenResponse(tokenResponse)
|
||||
const oauthTokenData = buildOAuthTokenData(tokenData, clientInfo)
|
||||
|
||||
this.saveTokens(oauthTokenData)
|
||||
return oauthTokenData
|
||||
}
|
||||
|
||||
async refresh(refreshToken: string): Promise<OAuthTokenData> {
|
||||
const metadata = await discoverOAuthServerMetadata(this.serverUrl)
|
||||
const clientInfo = this.clientInformation()
|
||||
const clientId = clientInfo?.clientId ?? this.configClientId
|
||||
if (!clientId) {
|
||||
throw new Error("No client information available. Run login() or register a client first.")
|
||||
}
|
||||
|
||||
const tokenData = (await tokenResponse.json()) as Record<string, unknown>
|
||||
const accessToken = tokenData.access_token
|
||||
if (typeof accessToken !== "string") {
|
||||
throw new Error("Token response missing access_token")
|
||||
}
|
||||
const tokenResponse = await fetch(metadata.tokenEndpoint, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/x-www-form-urlencoded" },
|
||||
body: new URLSearchParams({
|
||||
grant_type: "refresh_token",
|
||||
refresh_token: refreshToken,
|
||||
client_id: clientId,
|
||||
...(clientInfo?.clientSecret ? { client_secret: clientInfo.clientSecret } : {}),
|
||||
...(metadata.resource ? { resource: metadata.resource } : {}),
|
||||
}).toString(),
|
||||
})
|
||||
|
||||
const oauthTokenData: OAuthTokenData = {
|
||||
accessToken,
|
||||
refreshToken: typeof tokenData.refresh_token === "string" ? tokenData.refresh_token : undefined,
|
||||
expiresAt:
|
||||
typeof tokenData.expires_in === "number" ? Math.floor(Date.now() / 1000) + tokenData.expires_in : undefined,
|
||||
clientInfo: {
|
||||
clientId: clientInfo.clientId,
|
||||
clientSecret: clientInfo.clientSecret,
|
||||
},
|
||||
}
|
||||
const tokenData = await parseTokenResponse(tokenResponse)
|
||||
const oauthTokenData = buildOAuthTokenData(tokenData, {
|
||||
clientId,
|
||||
...(clientInfo?.clientSecret ? { clientSecret: clientInfo.clientSecret } : {}),
|
||||
}, refreshToken)
|
||||
|
||||
this.saveTokens(oauthTokenData)
|
||||
return oauthTokenData
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, afterAll, mock, spyOn } from "bun:test"
|
||||
import type { SkillMcpClientInfo, SkillMcpServerContext } from "./types"
|
||||
import type { ClaudeCodeMcpServer } from "../claude-code-mcp-loader/types"
|
||||
import type { OAuthTokenData } from "../mcp-oauth/storage"
|
||||
|
||||
// Mock the MCP SDK transports to avoid network calls
|
||||
const mockHttpConnect = mock(() => Promise.reject(new Error("Mocked HTTP connection failure")))
|
||||
const mockHttpClose = mock(() => Promise.resolve())
|
||||
let lastTransportInstance: { url?: URL; options?: { requestInit?: RequestInit } } = {}
|
||||
|
||||
const mockTokens = mock(() => null as { accessToken: string } | null)
|
||||
const mockLogin = mock(() => Promise.resolve({ accessToken: "test-token" }) as Promise<{ accessToken: string } | null>)
|
||||
const mockTokens = mock(() => null as OAuthTokenData | null)
|
||||
const mockLogin = mock(() => Promise.resolve({ accessToken: "test-token" } satisfies OAuthTokenData))
|
||||
const mockRefresh = mock((_: string) => Promise.resolve({ accessToken: "refreshed-token" } satisfies OAuthTokenData))
|
||||
|
||||
async function importFreshManagerModule(): Promise<typeof import("./manager")> {
|
||||
mock.module("@modelcontextprotocol/sdk/client/streamableHttp.js", () => ({
|
||||
@@ -41,12 +43,14 @@ describe("SkillMcpManager", () => {
|
||||
createOAuthProvider: () => ({
|
||||
tokens: () => mockTokens(),
|
||||
login: () => mockLogin(),
|
||||
refresh: (refreshToken: string) => mockRefresh(refreshToken),
|
||||
}),
|
||||
})
|
||||
mockHttpConnect.mockClear()
|
||||
mockHttpClose.mockClear()
|
||||
mockTokens.mockClear()
|
||||
mockLogin.mockClear()
|
||||
mockRefresh.mockClear()
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
@@ -724,6 +728,71 @@ describe("SkillMcpManager", () => {
|
||||
expect(headers?.Authorization).toBe("Bearer oauth-token")
|
||||
})
|
||||
|
||||
it("attempts silent refresh for expired stored tokens before login", async () => {
|
||||
// given
|
||||
const info: SkillMcpClientInfo = {
|
||||
serverName: "oauth-refresh",
|
||||
skillName: "oauth-skill",
|
||||
sessionID: "session-oauth-refresh",
|
||||
}
|
||||
const config: ClaudeCodeMcpServer = {
|
||||
url: "https://mcp.example.com/mcp",
|
||||
oauth: {
|
||||
clientId: "my-client",
|
||||
},
|
||||
}
|
||||
mockTokens.mockReturnValue({
|
||||
accessToken: "expired-token",
|
||||
refreshToken: "refresh-token",
|
||||
expiresAt: Math.floor(Date.now() / 1000) - 60,
|
||||
})
|
||||
mockRefresh.mockResolvedValue({ accessToken: "refreshed-token" })
|
||||
|
||||
// when
|
||||
try {
|
||||
await manager.getOrCreateClient(info, config)
|
||||
} catch { /* connection fails in test */ }
|
||||
|
||||
// then
|
||||
const headers = lastTransportInstance.options?.requestInit?.headers as Record<string, string> | undefined
|
||||
expect(headers?.Authorization).toBe("Bearer refreshed-token")
|
||||
expect(mockRefresh).toHaveBeenCalledWith("refresh-token")
|
||||
expect(mockLogin).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it("falls back to login when silent refresh fails", async () => {
|
||||
// given
|
||||
const info: SkillMcpClientInfo = {
|
||||
serverName: "oauth-refresh-fallback",
|
||||
skillName: "oauth-skill",
|
||||
sessionID: "session-oauth-refresh-fallback",
|
||||
}
|
||||
const config: ClaudeCodeMcpServer = {
|
||||
url: "https://mcp.example.com/mcp",
|
||||
oauth: {
|
||||
clientId: "my-client",
|
||||
},
|
||||
}
|
||||
mockTokens.mockReturnValue({
|
||||
accessToken: "expired-token",
|
||||
refreshToken: "refresh-token",
|
||||
expiresAt: Math.floor(Date.now() / 1000) - 60,
|
||||
})
|
||||
mockRefresh.mockRejectedValue(new Error("Refresh failed"))
|
||||
mockLogin.mockResolvedValue({ accessToken: "login-token" })
|
||||
|
||||
// when
|
||||
try {
|
||||
await manager.getOrCreateClient(info, config)
|
||||
} catch { /* connection fails in test */ }
|
||||
|
||||
// then
|
||||
const headers = lastTransportInstance.options?.requestInit?.headers as Record<string, string> | undefined
|
||||
expect(headers?.Authorization).toBe("Bearer login-token")
|
||||
expect(mockRefresh).toHaveBeenCalledWith("refresh-token")
|
||||
expect(mockLogin).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it("does not create auth provider when oauth config is absent", async () => {
|
||||
// given
|
||||
const info: SkillMcpClientInfo = {
|
||||
|
||||
@@ -44,7 +44,7 @@ export async function buildHttpRequestInit(
|
||||
const provider = getOrCreateAuthProvider(authProviders, config.url, config.oauth, createOAuthProvider)
|
||||
let tokenData = provider.tokens()
|
||||
|
||||
if (!tokenData || isTokenExpired(tokenData)) {
|
||||
if (!tokenData) {
|
||||
try {
|
||||
tokenData = await provider.login()
|
||||
} catch {
|
||||
@@ -52,6 +52,20 @@ export async function buildHttpRequestInit(
|
||||
}
|
||||
}
|
||||
|
||||
if (tokenData && isTokenExpired(tokenData)) {
|
||||
try {
|
||||
tokenData = tokenData.refreshToken
|
||||
? await provider.refresh(tokenData.refreshToken)
|
||||
: await provider.login()
|
||||
} catch {
|
||||
try {
|
||||
tokenData = await provider.login()
|
||||
} catch {
|
||||
tokenData = null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (tokenData) {
|
||||
headers.Authorization = `Bearer ${tokenData.accessToken}`
|
||||
}
|
||||
|
||||
@@ -50,7 +50,7 @@ export interface ProcessCleanupHandler {
|
||||
|
||||
export type OAuthProviderLike = Pick<
|
||||
McpOAuthProvider,
|
||||
"tokens" | "login"
|
||||
"tokens" | "login" | "refresh"
|
||||
>
|
||||
|
||||
export type OAuthProviderFactory = (options: {
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
/// <reference types="bun-types" />
|
||||
|
||||
import { afterAll, beforeEach, describe, expect, it, mock } from "bun:test"
|
||||
|
||||
const logMock = mock(() => {})
|
||||
|
||||
mock.module("../shared/logger", () => ({
|
||||
log: logMock,
|
||||
}))
|
||||
|
||||
afterAll(() => { mock.restore() })
|
||||
|
||||
const { createPreemptiveCompactionHook } = await import("./preemptive-compaction")
|
||||
|
||||
type AssistantHistoryMessage = {
|
||||
info: {
|
||||
id: string
|
||||
role: "assistant"
|
||||
}
|
||||
parts: Array<{ type: string; text?: string }>
|
||||
}
|
||||
|
||||
function createMockCtx(sessionHistory: AssistantHistoryMessage[]) {
|
||||
return {
|
||||
client: {
|
||||
session: {
|
||||
messages: mock(() => Promise.resolve({ data: sessionHistory })),
|
||||
summarize: mock(() => Promise.resolve({})),
|
||||
},
|
||||
tui: {
|
||||
showToast: mock(() => Promise.resolve({})),
|
||||
},
|
||||
},
|
||||
directory: "/tmp/test",
|
||||
}
|
||||
}
|
||||
|
||||
function appendAssistantHistory(
|
||||
sessionHistory: AssistantHistoryMessage[],
|
||||
input: {
|
||||
id: string
|
||||
parts: AssistantHistoryMessage["parts"]
|
||||
},
|
||||
): void {
|
||||
sessionHistory.push({
|
||||
info: {
|
||||
id: input.id,
|
||||
role: "assistant",
|
||||
},
|
||||
parts: input.parts,
|
||||
})
|
||||
}
|
||||
|
||||
function buildAssistantUpdate(input: {
|
||||
sessionID: string
|
||||
id: string
|
||||
parts: unknown[]
|
||||
}) {
|
||||
return {
|
||||
event: {
|
||||
type: "message.updated",
|
||||
properties: {
|
||||
info: {
|
||||
id: input.id,
|
||||
role: "assistant",
|
||||
sessionID: input.sessionID,
|
||||
providerID: "opencode",
|
||||
modelID: "kimi-k2.5-free",
|
||||
finish: true,
|
||||
tokens: { input: 1000, output: 10, reasoning: 0, cache: { read: 0, write: 0 } },
|
||||
parts: input.parts,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
describe("preemptive-compaction degradation monitor regressions", () => {
|
||||
beforeEach(() => {
|
||||
logMock.mockClear()
|
||||
})
|
||||
|
||||
it("does not re-arm monitoring after recovery-triggered compaction", async () => {
|
||||
// given
|
||||
const sessionHistory: AssistantHistoryMessage[] = []
|
||||
const ctx = createMockCtx(sessionHistory)
|
||||
const hook = createPreemptiveCompactionHook(ctx as never, {} as never)
|
||||
const sessionID = "ses_recovery_compaction_guard"
|
||||
const stepOnlyParts = [{ type: "step-start" }, { type: "step-finish" }]
|
||||
|
||||
await hook.event({
|
||||
event: {
|
||||
type: "session.compacted",
|
||||
properties: { sessionID },
|
||||
},
|
||||
})
|
||||
|
||||
// when
|
||||
appendAssistantHistory(sessionHistory, { id: "msg_1", parts: stepOnlyParts })
|
||||
await hook.event(buildAssistantUpdate({ sessionID, id: "msg_1", parts: stepOnlyParts }))
|
||||
|
||||
appendAssistantHistory(sessionHistory, { id: "msg_2", parts: stepOnlyParts })
|
||||
await hook.event(buildAssistantUpdate({ sessionID, id: "msg_2", parts: stepOnlyParts }))
|
||||
|
||||
appendAssistantHistory(sessionHistory, { id: "msg_3", parts: stepOnlyParts })
|
||||
await hook.event(buildAssistantUpdate({ sessionID, id: "msg_3", parts: stepOnlyParts }))
|
||||
|
||||
await hook.event({
|
||||
event: {
|
||||
type: "session.compacted",
|
||||
properties: { sessionID },
|
||||
},
|
||||
})
|
||||
|
||||
appendAssistantHistory(sessionHistory, { id: "msg_4", parts: stepOnlyParts })
|
||||
await hook.event(buildAssistantUpdate({ sessionID, id: "msg_4", parts: stepOnlyParts }))
|
||||
|
||||
appendAssistantHistory(sessionHistory, { id: "msg_5", parts: stepOnlyParts })
|
||||
await hook.event(buildAssistantUpdate({ sessionID, id: "msg_5", parts: stepOnlyParts }))
|
||||
|
||||
appendAssistantHistory(sessionHistory, { id: "msg_6", parts: stepOnlyParts })
|
||||
await hook.event(buildAssistantUpdate({ sessionID, id: "msg_6", parts: stepOnlyParts }))
|
||||
|
||||
// then
|
||||
expect(ctx.client.session.summarize).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
@@ -6,6 +6,7 @@ import { resolveCompactionModel } from "./shared/compaction-model-resolver"
|
||||
const PREEMPTIVE_COMPACTION_TIMEOUT_MS = 120_000
|
||||
const POST_COMPACTION_MONITOR_COUNT = 5
|
||||
const POST_COMPACTION_NO_TEXT_THRESHOLD = 3
|
||||
const RECOVERY_COMPACTION_SUPPRESSION_MS = 5_000
|
||||
|
||||
declare function setTimeout(handler: () => void, timeout?: number): unknown
|
||||
declare function clearTimeout(timeoutID: unknown): void
|
||||
@@ -74,6 +75,7 @@ export function createPostCompactionDegradationMonitor(args: {
|
||||
const postCompactionNoTextStreak = new Map<string, number>()
|
||||
const postCompactionRecoveryTriggered = new Set<string>()
|
||||
const postCompactionEpoch = new Map<string, number>()
|
||||
const suppressRecoveryCompactionUntil = new Map<string, number>()
|
||||
const postCompactionRecoveryCount = new Map<string, number>()
|
||||
|
||||
const MAX_RECOVERY_ATTEMPTS = 3
|
||||
@@ -87,6 +89,13 @@ export function createPostCompactionDegradationMonitor(args: {
|
||||
}
|
||||
|
||||
const onSessionCompacted = (sessionID: string): void => {
|
||||
const suppressedUntil = suppressRecoveryCompactionUntil.get(sessionID)
|
||||
if (suppressedUntil && suppressedUntil > Date.now()) {
|
||||
suppressRecoveryCompactionUntil.delete(sessionID)
|
||||
return
|
||||
}
|
||||
suppressRecoveryCompactionUntil.delete(sessionID)
|
||||
|
||||
const nextEpoch = (postCompactionEpoch.get(sessionID) ?? 0) + 1
|
||||
postCompactionEpoch.set(sessionID, nextEpoch)
|
||||
postCompactionRemaining.set(sessionID, POST_COMPACTION_MONITOR_COUNT)
|
||||
@@ -116,6 +125,7 @@ export function createPostCompactionDegradationMonitor(args: {
|
||||
postCompactionRecoveryTriggered.add(sessionID)
|
||||
compactionInProgress.add(sessionID)
|
||||
const recoveryEpoch = postCompactionEpoch.get(sessionID) ?? 0
|
||||
suppressRecoveryCompactionUntil.set(sessionID, Date.now() + RECOVERY_COMPACTION_SUPPRESSION_MS)
|
||||
|
||||
try {
|
||||
const { providerID: targetProviderID, modelID: targetModelID } = resolveCompactionModel(
|
||||
@@ -148,6 +158,7 @@ export function createPostCompactionDegradationMonitor(args: {
|
||||
|
||||
log("[preemptive-compaction] Triggered recovery after post-compaction no-text tail", { sessionID })
|
||||
} catch (error) {
|
||||
suppressRecoveryCompactionUntil.delete(sessionID)
|
||||
log("[preemptive-compaction] Failed to recover post-compaction no-text tail", {
|
||||
sessionID,
|
||||
error: String(error),
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
export interface AutoRetrySignal {
|
||||
signal: string
|
||||
}
|
||||
|
||||
const AUTO_RETRY_PATTERNS: Array<(combined: string) => boolean> = [
|
||||
(combined) => /retrying\s+in/i.test(combined),
|
||||
(combined) =>
|
||||
/(?:too\s+many\s+requests|quota\s+will\s+reset\s+after|quota\s*exceeded|usage\s+limit|rate\s+limit|limit\s+reached|all\s+credentials\s+for\s+model|cool(?:ing)?\s*down|exhausted\s+your\s+capacity)/i.test(combined),
|
||||
]
|
||||
|
||||
export function extractAutoRetrySignal(info: Record<string, unknown> | undefined): AutoRetrySignal | undefined {
|
||||
if (!info) return undefined
|
||||
|
||||
const candidates: string[] = []
|
||||
|
||||
const directStatus = info.status
|
||||
if (typeof directStatus === "string") candidates.push(directStatus)
|
||||
|
||||
const summary = info.summary
|
||||
if (typeof summary === "string") candidates.push(summary)
|
||||
|
||||
const message = info.message
|
||||
if (typeof message === "string") candidates.push(message)
|
||||
|
||||
const details = info.details
|
||||
if (typeof details === "string") candidates.push(details)
|
||||
|
||||
const combined = candidates.join("\n")
|
||||
if (!combined) return undefined
|
||||
|
||||
return AUTO_RETRY_PATTERNS.some((test) => test(combined)) ? { signal: combined } : undefined
|
||||
}
|
||||
@@ -11,7 +11,7 @@ import type { RuntimeFallbackConfig } from "../../config"
|
||||
*/
|
||||
export const DEFAULT_CONFIG: Required<RuntimeFallbackConfig> = {
|
||||
enabled: false,
|
||||
retry_on_errors: [402, 429, 500, 502, 503, 504],
|
||||
retry_on_errors: [429, 500, 502, 503, 504],
|
||||
max_fallback_attempts: 3,
|
||||
cooldown_seconds: 60,
|
||||
timeout_seconds: 30,
|
||||
@@ -25,26 +25,21 @@ export const DEFAULT_CONFIG: Required<RuntimeFallbackConfig> = {
|
||||
export const RETRYABLE_ERROR_PATTERNS = [
|
||||
/rate.?limit/i,
|
||||
/too.?many.?requests/i,
|
||||
/quota.?exceeded/i,
|
||||
/quota\s+will\s+reset\s+after/i,
|
||||
/quota.?exceeded/i,
|
||||
/(?:you(?:'ve|\s+have)\s+)?reached\s+your\s+usage\s+limit/i,
|
||||
/all\s+credentials\s+for\s+model/i,
|
||||
/cool(?:ing)?\s+down/i,
|
||||
/exhausted\s+your\s+capacity/i,
|
||||
/usage\s+limit\s+has\s+been\s+reached/i,
|
||||
/all\s+credentials\s+for\s+model/i,
|
||||
/cool(?:ing)?\s+down/i,
|
||||
/model.{0,20}?not.{0,10}?supported/i,
|
||||
/model_not_supported/i,
|
||||
/insufficient.?(?:credits?|funds?|balance)/i,
|
||||
/credit.*balance.*too.*low/i,
|
||||
/service.?unavailable/i,
|
||||
/overloaded/i,
|
||||
/temporarily.?unavailable/i,
|
||||
/try.?again/i,
|
||||
/credit.*balance.*too.*low/i,
|
||||
/insufficient.?(?:credits?|funds?|balance)/i,
|
||||
/subscription.*quota/i,
|
||||
/billing.?(?:hard.?)?limit/i,
|
||||
/payment.?required/i,
|
||||
/out\s+of\s+credits?/i,
|
||||
/(?:^|\s)402(?:\s|$)/,
|
||||
/(?:^|\s)429(?:\s|$)/,
|
||||
/(?:^|\s)503(?:\s|$)/,
|
||||
/(?:^|\s)529(?:\s|$)/,
|
||||
|
||||
@@ -181,113 +181,7 @@ describe("extractStatusCode", () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe("quota error detection (fixes #2747)", () => {
|
||||
test("classifies prettified subscription quota error as quota_exceeded", () => {
|
||||
//#given
|
||||
const error = {
|
||||
name: "AI_APICallError",
|
||||
message: "Subscription quota exceeded. You can continue using free models.",
|
||||
}
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
const retryable = isRetryableError(error, [402, 429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
test("classifies billing hard limit error as quota_exceeded", () => {
|
||||
//#given
|
||||
const error = { message: "You have reached your billing hard limit." }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
})
|
||||
|
||||
test("classifies exhausted capacity error as quota_exceeded", () => {
|
||||
//#given
|
||||
const error = { message: "You have exhausted your capacity on this model." }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
})
|
||||
|
||||
test("classifies out of credits error as quota_exceeded", () => {
|
||||
//#given
|
||||
const error = { message: "Out of credits. Please add more credits to continue." }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
})
|
||||
|
||||
test("treats HTTP 402 Payment Required as retryable", () => {
|
||||
//#given
|
||||
const error = { statusCode: 402, message: "Payment Required" }
|
||||
|
||||
//#when
|
||||
const retryable = isRetryableError(error, [402, 429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
test("matches subscription quota pattern in RETRYABLE_ERROR_PATTERNS", () => {
|
||||
//#given
|
||||
const error = { message: "Subscription quota exceeded. You can continue using free models." }
|
||||
|
||||
//#when
|
||||
const retryable = isRetryableError(error, [429, 503])
|
||||
|
||||
//#then
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
test("treats hard usage-limit wording as retryable", () => {
|
||||
//#given
|
||||
const error = { message: "You've reached your usage limit for this month. Please upgrade to continue." }
|
||||
|
||||
//#when
|
||||
const retryable = isRetryableError(error, [429, 503])
|
||||
|
||||
//#then
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
test("classifies QuotaExceededError by errorName even without quota keywords in message", () => {
|
||||
//#given
|
||||
const error = { name: "QuotaExceededError", message: "Request failed." }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
})
|
||||
|
||||
test("detects payment required errors as retryable", () => {
|
||||
//#given
|
||||
const error = { message: "Error 402: payment required for this request" }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
const retryable = isRetryableError(error, [429, 503])
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
describe("model support fallback", () => {
|
||||
test("detects model_not_supported errors as retryable for fallback chain", () => {
|
||||
//#given
|
||||
const error1 = { message: "model_not_supported" }
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { DEFAULT_CONFIG, RETRYABLE_ERROR_PATTERNS } from "./constants"
|
||||
|
||||
export { extractAutoRetrySignal } from "./auto-retry-signal"
|
||||
|
||||
export function getErrorMessage(error: unknown): string {
|
||||
if (!error) return ""
|
||||
if (typeof error === "string") return error.toLowerCase()
|
||||
@@ -137,44 +139,6 @@ export function classifyErrorType(error: unknown): string | undefined {
|
||||
return undefined
|
||||
}
|
||||
|
||||
export interface AutoRetrySignal {
|
||||
signal: string
|
||||
}
|
||||
|
||||
export const AUTO_RETRY_PATTERNS: Array<(combined: string) => boolean> = [
|
||||
(combined) => /retrying\s+in/i.test(combined),
|
||||
(combined) =>
|
||||
/(?:too\s+many\s+requests|quota\s*exceeded|quota\s+will\s+reset\s+after|usage\s+limit|rate\s+limit|limit\s+reached|all\s+credentials\s+for\s+model|cool(?:ing)?\s*down|exhausted\s+your\s+capacity)/i.test(combined),
|
||||
]
|
||||
|
||||
export function extractAutoRetrySignal(info: Record<string, unknown> | undefined): AutoRetrySignal | undefined {
|
||||
if (!info) return undefined
|
||||
|
||||
const candidates: string[] = []
|
||||
|
||||
const directStatus = info.status
|
||||
if (typeof directStatus === "string") candidates.push(directStatus)
|
||||
|
||||
const summary = info.summary
|
||||
if (typeof summary === "string") candidates.push(summary)
|
||||
|
||||
const message = info.message
|
||||
if (typeof message === "string") candidates.push(message)
|
||||
|
||||
const details = info.details
|
||||
if (typeof details === "string") candidates.push(details)
|
||||
|
||||
const combined = candidates.join("\n")
|
||||
if (!combined) return undefined
|
||||
|
||||
const isAutoRetry = AUTO_RETRY_PATTERNS.some((test) => test(combined))
|
||||
if (isAutoRetry) {
|
||||
return { signal: combined }
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
export function containsErrorContent(
|
||||
parts: Array<{ type?: string; text?: string }> | undefined
|
||||
): { hasError: boolean; errorMessage?: string } {
|
||||
@@ -204,7 +168,10 @@ export function isRetryableError(error: unknown, retryOnErrors: number[]): boole
|
||||
}
|
||||
|
||||
if (errorType === "quota_exceeded") {
|
||||
return true
|
||||
// When a provider signals an auto-retry (e.g. "retrying in ~2 weeks"),
|
||||
// we should still trigger fallback to another model rather than STOP.
|
||||
const hasAutoRetrySignal = /retrying\s+in/i.test(message)
|
||||
return hasAutoRetrySignal
|
||||
}
|
||||
|
||||
if (statusCode && retryOnErrors.includes(statusCode)) {
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
|
||||
import { classifyErrorType, isRetryableError } from "./error-classifier"
|
||||
|
||||
describe("runtime-fallback quota error regressions", () => {
|
||||
test("classifies subscription quota errors as quota_exceeded and stops retry", () => {
|
||||
//#given
|
||||
const error = {
|
||||
name: "AI_APICallError",
|
||||
message: "Subscription quota exceeded. You can continue using free models.",
|
||||
}
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
const retryable = isRetryableError(error, [429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
expect(retryable).toBe(false)
|
||||
})
|
||||
|
||||
test("treats HTTP 402 payment required as non-retryable", () => {
|
||||
//#given
|
||||
const error = { statusCode: 402, message: "Payment Required" }
|
||||
|
||||
//#when
|
||||
const retryable = isRetryableError(error, [429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(retryable).toBe(false)
|
||||
})
|
||||
|
||||
test("keeps HTTP 429 rate limit retryable", () => {
|
||||
//#given
|
||||
const error = { statusCode: 429, message: "Too Many Requests: rate limit reached" }
|
||||
|
||||
//#when
|
||||
const retryable = isRetryableError(error, [429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(retryable).toBe(true)
|
||||
})
|
||||
|
||||
test("classifies quota error names as quota_exceeded without retry", () => {
|
||||
//#given
|
||||
const error = { name: "QuotaExceededError", message: "Request failed." }
|
||||
|
||||
//#when
|
||||
const errorType = classifyErrorType(error)
|
||||
const retryable = isRetryableError(error, [429, 500, 502, 503, 504])
|
||||
|
||||
//#then
|
||||
expect(errorType).toBe("quota_exceeded")
|
||||
expect(retryable).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -215,6 +215,28 @@ describe("model-error-classifier", () => {
|
||||
//#then
|
||||
expect(result).toBe(true)
|
||||
})
|
||||
|
||||
test("treats subscription quota message as non-retryable", () => {
|
||||
//#given
|
||||
const error = { message: "Subscription quota exceeded. You can continue using free models." }
|
||||
|
||||
//#when
|
||||
const result = shouldRetryError(error)
|
||||
|
||||
//#then
|
||||
expect(result).toBe(false)
|
||||
})
|
||||
|
||||
test("treats HTTP 429 rate limit message as retryable", () => {
|
||||
//#given
|
||||
const error = { message: "429 Too Many Requests: rate limit reached" }
|
||||
|
||||
//#when
|
||||
const result = shouldRetryError(error)
|
||||
|
||||
//#then
|
||||
expect(result).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
export {}
|
||||
|
||||
@@ -84,23 +84,25 @@ const STOP_MESSAGE_PATTERNS = [
|
||||
"usage limit has been reached",
|
||||
"free usage limit",
|
||||
"billing limit",
|
||||
"billing hard limit",
|
||||
"monthly limit",
|
||||
"plan limit",
|
||||
"subscription quota",
|
||||
"subscription limit",
|
||||
"payment required",
|
||||
"out of credits",
|
||||
"credits exhausted",
|
||||
"insufficient credits",
|
||||
"insufficient balance",
|
||||
"credit balance",
|
||||
"usage limit for this month",
|
||||
"exhausted your capacity",
|
||||
]
|
||||
|
||||
const AUTO_RETRY_GATE_PATTERNS = [
|
||||
"rate limit",
|
||||
"quota",
|
||||
"usage limit",
|
||||
"limit reached",
|
||||
"cooling down",
|
||||
"credentials for model",
|
||||
"exhausted your capacity",
|
||||
]
|
||||
|
||||
function hasProviderAutoRetrySignal(message: string): boolean {
|
||||
|
||||
Reference in New Issue
Block a user