diff --git a/src/features/background-agent/error-classifier.ts b/src/features/background-agent/error-classifier.ts index 523f61bc1..7fbfd031b 100644 --- a/src/features/background-agent/error-classifier.ts +++ b/src/features/background-agent/error-classifier.ts @@ -65,6 +65,33 @@ export function extractErrorMessage(error: unknown): string | undefined { } } +export function extractErrorStatusCode(error: unknown): number | undefined { + if (!isRecord(error)) return undefined + + for (const key of ["statusCode", "status", "code"]) { + const val = (error as Record)[key] + if (typeof val === "number" && val >= 100 && val < 600) return val + } + + const statusVal = (error as Record)["status"] + if (typeof statusVal === "string") { + const parsed = parseInt(statusVal, 10) + if (parsed >= 100 && parsed < 600) return parsed + } + + const responseRaw = (error as Record)["response"] + if (isRecord(responseRaw)) { + const respStatus = responseRaw["status"] + if (typeof respStatus === "number" && respStatus >= 100 && respStatus < 600) return respStatus + if (typeof respStatus === "string") { + const parsed = parseInt(respStatus, 10) + if (parsed >= 100 && parsed < 600) return parsed + } + } + + return undefined +} + interface EventPropertiesLike { [key: string]: unknown } diff --git a/src/features/background-agent/fallback-retry-handler.ts b/src/features/background-agent/fallback-retry-handler.ts index f5f31a6ad..ab8bba91a 100644 --- a/src/features/background-agent/fallback-retry-handler.ts +++ b/src/features/background-agent/fallback-retry-handler.ts @@ -19,7 +19,7 @@ function canonicalizeModelID(modelID: string): string { export async function tryFallbackRetry(args: { task: BackgroundTask - errorInfo: { name?: string; message?: string } + errorInfo: { name?: string; message?: string; statusCode?: number } source: string concurrencyManager: ConcurrencyManager client: OpencodeClient diff --git a/src/features/background-agent/manager.ts b/src/features/background-agent/manager.ts index 9442c23d9..d033b350c 100644 --- a/src/features/background-agent/manager.ts +++ b/src/features/background-agent/manager.ts @@ -46,6 +46,7 @@ import { isAbortedSessionError, extractErrorName, extractErrorMessage, + extractErrorStatusCode, getSessionErrorMessage, isRecord, } from "./error-classifier" @@ -749,6 +750,7 @@ The fallback retry session is now created and can be inspected directly. const errorInfo = { name: extractErrorName(error), message: extractErrorMessage(error), + statusCode: extractErrorStatusCode(error), } if (await this.tryFallbackRetry(existingTask, errorInfo, "promptAsync.launch")) { return @@ -1079,6 +1081,7 @@ The fallback retry session is now created and can be inspected directly. const errorInfo = { name: extractErrorName(error), message: extractErrorMessage(error), + statusCode: extractErrorStatusCode(error), } if (await this.tryFallbackRetry(existingTask, errorInfo, "promptAsync.resume")) { return @@ -1199,6 +1202,7 @@ The fallback retry session is now created and can be inspected directly. const errorInfo = { name: extractErrorName(assistantError), message: extractErrorMessage(assistantError), + statusCode: extractErrorStatusCode(assistantError), } void this.tryFallbackRetry(task, errorInfo, "message.updated").catch((error) => { log("[background-agent] Error handling message.updated fallback retry:", { @@ -1444,7 +1448,7 @@ The fallback retry session is now created and can be inspected directly. private async handleSessionErrorEvent(args: { task: BackgroundTask - errorInfo: { name?: string; message?: string } + errorInfo: { name?: string; message?: string; statusCode?: number } errorName: string | undefined errorMessage: string | undefined }): Promise { @@ -1532,7 +1536,7 @@ The fallback retry session is now created and can be inspected directly. private tryFallbackRetry( task: BackgroundTask, - errorInfo: { name?: string; message?: string }, + errorInfo: { name?: string; message?: string; statusCode?: number }, source: string, ): Promise { const previousSessionID = task.sessionId diff --git a/src/shared/model-error-classifier.test.ts b/src/shared/model-error-classifier.test.ts index c7f41ef4e..33fd3f4c1 100644 --- a/src/shared/model-error-classifier.test.ts +++ b/src/shared/model-error-classifier.test.ts @@ -3,7 +3,7 @@ const { describe, expect, test, beforeEach, afterEach, mock, spyOn } = require(" import * as connectedProvidersCache from "./connected-providers-cache" let readConnectedProvidersCacheSpy: ReturnType | undefined -const { shouldRetryError, selectFallbackProvider } = await import("./model-error-classifier") +const { shouldRetryError, selectFallbackProvider, isRetryableModelError } = await import("./model-error-classifier") describe("model-error-classifier", () => { beforeEach(() => { @@ -270,6 +270,139 @@ describe("model-error-classifier", () => { //#then expect(result).toBe(false) }) + + test("GLM 429 rate limit with statusCode and Chinese message triggers fallback (statusCode check)", () => { + //#given + const error = { statusCode: 429, message: "请求频率过高" } + + //#when + const result = isRetryableModelError(error) + + //#then + expect(result).toBe(true) + }) + + test("GLM 429 rate limit with statusCode and no message at all triggers fallback", () => { + //#given + const error = { statusCode: 429 } + + //#when + const result = isRetryableModelError(error) + + //#then + expect(result).toBe(true) + }) + + test("GLM 503 service unavailable with statusCode triggers fallback", () => { + //#given + const error = { statusCode: 503, message: "Service Unavailable" } + + //#when + const result = isRetryableModelError(error) + + //#then + expect(result).toBe(true) + }) + + test("GLM 529 overloaded with statusCode triggers fallback", () => { + //#given + const error = { statusCode: 529 } + + //#when + const result = isRetryableModelError(error) + + //#then + expect(result).toBe(true) + }) + + test("HTTP 400 with statusCode does NOT trigger fallback via statusCode alone (400 excluded)", () => { + //#given — message does NOT match any retryable pattern + const error = { statusCode: 400, message: "Invalid parameter: model_name" } + + //#when + const result = isRetryableModelError(error) + + //#then + expect(result).toBe(false) + }) + + test("HTTP 401 with statusCode does NOT trigger fallback (not a rate limit)", () => { + //#given + const error = { statusCode: 401, message: "Unauthorized" } + + //#when + const result = isRetryableModelError(error) + + //#then + expect(result).toBe(false) + }) + + test("GLM code 1304 daily quota 429 does NOT trigger fallback (STOP pattern wins)", () => { + //#given + const error = { + statusCode: 429, + message: "Daily call limit for this API key has been reached. Limit will reset at midnight UTC.", + } + + //#when + const result = isRetryableModelError(error) + + //#then + expect(result).toBe(false) + }) + + test("GLM account in arrears 429 does NOT trigger fallback (STOP pattern wins)", () => { + //#given + const error = { + statusCode: 429, + message: "Your account is in arrears, please recharge and try again.", + } + + //#when + const result = isRetryableModelError(error) + + //#then + expect(result).toBe(false) + }) + + test("GLM fair use policy violation 429 does NOT trigger fallback (STOP pattern wins)", () => { + //#given + const error = { + statusCode: 429, + message: "Request blocked under Fair Use Policy. Your request rate has been restricted.", + } + + //#when + const result = isRetryableModelError(error) + + //#then + expect(result).toBe(false) + }) + + test("STOP message pattern takes precedence over 429 statusCode", () => { + //#given + const error = { + statusCode: 429, + message: "quota exceeded for this account, usage limit has been reached", + } + + //#when + const result = isRetryableModelError(error) + + //#then + expect(result).toBe(false) + }) + + test("rate limit message without statusCode still works (backward compat)", () => { + //#given + const error = { message: "rate limit reached for requests" } + + //#when + const result = isRetryableModelError(error) + + //#then + expect(result).toBe(true) + }) }) export {} diff --git a/src/shared/model-error-classifier.ts b/src/shared/model-error-classifier.ts index 2c74d0831..fb3fdf480 100644 --- a/src/shared/model-error-classifier.ts +++ b/src/shared/model-error-classifier.ts @@ -99,6 +99,13 @@ const STOP_MESSAGE_PATTERNS = [ "credit balance", "usage limit for this month", "exhausted your capacity", + // GLM/Z.ai business error codes that indicate permanent quota/billing exhaustion + "daily call limit", + "daily limit", + "usage limit reached for", + "in arrears", + "fair use policy", + "recharge and try", ] const AUTO_RETRY_GATE_PATTERNS = [ @@ -117,6 +124,8 @@ function hasProviderAutoRetrySignal(message: string): boolean { export interface ErrorInfo { name?: string message?: string + /** HTTP status code from the provider response (e.g., 429 for rate limit) */ + statusCode?: number } /** @@ -151,6 +160,16 @@ export function isRetryableModelError(error: ErrorInfo): boolean { if (hasProviderAutoRetrySignal(msg)) { return true } + + // HTTP status code check: catches rate-limit errors regardless of message format/language. + // Uses the same codes as runtime-fallback config (400 excluded as it is a permanent client error). + if ( + error.statusCode != null && + (error.statusCode === 429 || error.statusCode === 503 || error.statusCode === 529) + ) { + return true + } + return RETRYABLE_MESSAGE_PATTERNS.some((pattern) => msg.includes(pattern)) }