Fix model fallback across main/background/sync agents

This commit is contained in:
VespianRex
2026-02-19 04:41:00 +02:00
parent d8da89fd5b
commit f5f1d1d4c2
44 changed files with 2500 additions and 605 deletions
+8
View File
@@ -75,6 +75,14 @@ function findVariantInChain(
return entry.variant
}
}
// Some providers expose identical model IDs (e.g. OpenAI models via different providers).
// If we didn't find an exact provider+model match, fall back to model-only matching.
for (const entry of fallbackChain) {
if (entry.model === currentModel.modelID) {
return entry.variant
}
}
return undefined
}
+76
View File
@@ -0,0 +1,76 @@
import { describe, expect, test, beforeEach, afterEach, spyOn } from "bun:test"
import { mkdirSync, rmSync, writeFileSync, existsSync } from "node:fs"
import { join } from "node:path"
import * as dataPath from "./data-path"
import { shouldRetryError, selectFallbackProvider } from "./model-error-classifier"
const TEST_CACHE_DIR = join(import.meta.dir, "__test-cache__")
describe("model-error-classifier", () => {
let cacheDirSpy: ReturnType<typeof spyOn>
beforeEach(() => {
cacheDirSpy = spyOn(dataPath, "getOmoOpenCodeCacheDir").mockReturnValue(TEST_CACHE_DIR)
if (existsSync(TEST_CACHE_DIR)) {
rmSync(TEST_CACHE_DIR, { recursive: true })
}
mkdirSync(TEST_CACHE_DIR, { recursive: true })
})
afterEach(() => {
cacheDirSpy.mockRestore()
if (existsSync(TEST_CACHE_DIR)) {
rmSync(TEST_CACHE_DIR, { recursive: true })
}
})
test("treats overloaded retry messages as retryable", () => {
//#given
const error = { message: "Provider is overloaded" }
//#when
const result = shouldRetryError(error)
//#then
expect(result).toBe(true)
})
test("selectFallbackProvider prefers first connected provider in preference order", () => {
//#given
writeFileSync(
join(TEST_CACHE_DIR, "connected-providers.json"),
JSON.stringify({ connected: ["quotio", "nvidia"], updatedAt: new Date().toISOString() }, null, 2),
)
//#when
const provider = selectFallbackProvider(["quotio", "nvidia"], "nvidia")
//#then
expect(provider).toBe("quotio")
})
test("selectFallbackProvider falls back to next connected provider when first is disconnected", () => {
//#given
writeFileSync(
join(TEST_CACHE_DIR, "connected-providers.json"),
JSON.stringify({ connected: ["nvidia"], updatedAt: new Date().toISOString() }, null, 2),
)
//#when
const provider = selectFallbackProvider(["quotio", "nvidia"])
//#then
expect(provider).toBe("nvidia")
})
test("selectFallbackProvider uses provider preference order when cache is missing", () => {
//#given - no cache file
//#when
const provider = selectFallbackProvider(["quotio", "nvidia"], "nvidia")
//#then
expect(provider).toBe("quotio")
})
})
+135
View File
@@ -0,0 +1,135 @@
import type { FallbackEntry } from "./model-requirements"
import { readConnectedProvidersCache } from "./connected-providers-cache"
/**
* Error names that indicate a retryable model error (deadstop).
* These errors completely halt the action loop and should trigger fallback retry.
*/
const RETRYABLE_ERROR_NAMES = new Set([
"ProviderModelNotFoundError",
"RateLimitError",
"QuotaExceededError",
"InsufficientCreditsError",
"ModelUnavailableError",
"ProviderConnectionError",
"AuthenticationError",
])
/**
* Error names that should NOT trigger retry.
* These errors are typically user-induced or fixable without switching models.
*/
const NON_RETRYABLE_ERROR_NAMES = new Set([
"MessageAbortedError",
"PermissionDeniedError",
"ContextLengthError",
"TimeoutError",
"ValidationError",
"SyntaxError",
"UserError",
])
/**
* Message patterns that indicate a retryable error even without a known error name.
*/
const RETRYABLE_MESSAGE_PATTERNS = [
"rate_limit",
"rate limit",
"quota",
"not found",
"unavailable",
"insufficient",
"too many requests",
"over limit",
"overloaded",
"bad gateway",
"unknown provider",
"provider not found",
"connection error",
"network error",
"timeout",
"service unavailable",
"internal_server_error",
"503",
"502",
"504",
]
export interface ErrorInfo {
name?: string
message?: string
}
/**
* Determines if an error is a retryable model error.
* Returns true if the error is a known retryable type OR matches retryable message patterns.
*/
export function isRetryableModelError(error: ErrorInfo): boolean {
// If we have an error name, check against known lists
if (error.name) {
// Explicit non-retryable takes precedence
if (NON_RETRYABLE_ERROR_NAMES.has(error.name)) {
return false
}
// Check if it's a known retryable error
if (RETRYABLE_ERROR_NAMES.has(error.name)) {
return true
}
}
// Check message patterns for unknown errors
const msg = error.message?.toLowerCase() ?? ""
return RETRYABLE_MESSAGE_PATTERNS.some((pattern) => msg.includes(pattern))
}
/**
* Determines if an error should trigger a fallback retry.
* Returns true for deadstop errors that completely halt the action loop.
*/
export function shouldRetryError(error: ErrorInfo): boolean {
return isRetryableModelError(error)
}
/**
* Gets the next fallback model from the chain based on attempt count.
* Returns undefined if all fallbacks have been exhausted.
*/
export function getNextFallback(
fallbackChain: FallbackEntry[],
attemptCount: number,
): FallbackEntry | undefined {
return fallbackChain[attemptCount]
}
/**
* Checks if there are more fallbacks available after the current attempt.
*/
export function hasMoreFallbacks(
fallbackChain: FallbackEntry[],
attemptCount: number,
): boolean {
return attemptCount < fallbackChain.length
}
/**
* Selects the best provider for a fallback entry.
* Priority:
* 1) First connected provider in the entry's provider preference order
* 2) First provider listed in the fallback entry (when cache is missing)
*/
export function selectFallbackProvider(
providers: string[],
preferredProviderID?: string,
): string {
const connectedProviders = readConnectedProvidersCache()
if (connectedProviders) {
const connectedSet = new Set(connectedProviders)
for (const provider of providers) {
if (connectedSet.has(provider)) {
return provider
}
}
}
return providers[0] || preferredProviderID || "quotio"
}
+128 -463
View File
@@ -6,493 +6,158 @@ import {
type ModelRequirement,
} from "./model-requirements"
function flattenChains(): FallbackEntry[] {
return [
...Object.values(AGENT_MODEL_REQUIREMENTS).flatMap((r) => r.fallbackChain),
...Object.values(CATEGORY_MODEL_REQUIREMENTS).flatMap((r) => r.fallbackChain),
]
}
function assertNoExcludedModels(entry: FallbackEntry): void {
// User exclusions.
expect(entry.model).not.toBe("grok-code-fast-1")
if (entry.providers.includes("quotio")) {
expect(entry.model).not.toBe("tstars2.0")
expect(entry.model).not.toMatch(/^kiro-/i)
expect(entry.model).not.toMatch(/^tab_/i)
}
// Remove codex-mini models per request.
expect(entry.model).not.toMatch(/codex-mini/i)
}
function assertNoOpencodeProvider(entry: FallbackEntry): void {
expect(entry.providers).not.toContain("opencode")
}
function assertNoProviderPrefixForNonNamespacedProviders(entry: FallbackEntry): void {
// For these providers, model IDs should not be written as "provider/model".
const nonNamespaced = ["quotio", "openai", "github-copilot", "minimax", "minimax-coding-plan"]
for (const provider of entry.providers) {
if (!nonNamespaced.includes(provider)) continue
expect(entry.model.startsWith(`${provider}/`)).toBe(false)
}
}
describe("AGENT_MODEL_REQUIREMENTS", () => {
test("oracle has valid fallbackChain with gpt-5.2 as primary", () => {
// given - oracle agent requirement
const oracle = AGENT_MODEL_REQUIREMENTS["oracle"]
// when - accessing oracle requirement
// then - fallbackChain exists with gpt-5.2 as first entry
expect(oracle).toBeDefined()
expect(oracle.fallbackChain).toBeArray()
expect(oracle.fallbackChain.length).toBeGreaterThan(0)
const primary = oracle.fallbackChain[0]
expect(primary.providers).toContain("openai")
expect(primary.model).toBe("gpt-5.2")
expect(primary.variant).toBe("high")
})
test("sisyphus has claude-opus-4-6 as primary and requiresAnyModel", () => {
// #given - sisyphus agent requirement
const sisyphus = AGENT_MODEL_REQUIREMENTS["sisyphus"]
// #when - accessing Sisyphus requirement
// #then - fallbackChain has claude-opus-4-6 first, big-pickle last
expect(sisyphus).toBeDefined()
expect(sisyphus.fallbackChain).toBeArray()
expect(sisyphus.fallbackChain).toHaveLength(5)
expect(sisyphus.requiresAnyModel).toBe(true)
const primary = sisyphus.fallbackChain[0]
expect(primary.providers).toEqual(["anthropic", "github-copilot", "opencode"])
expect(primary.model).toBe("claude-opus-4-6")
expect(primary.variant).toBe("max")
const last = sisyphus.fallbackChain[4]
expect(last.providers[0]).toBe("opencode")
expect(last.model).toBe("big-pickle")
})
test("librarian has valid fallbackChain with gemini-3-flash as primary", () => {
// given - librarian agent requirement
const librarian = AGENT_MODEL_REQUIREMENTS["librarian"]
// when - accessing librarian requirement
// then - fallbackChain exists with gemini-3-flash as first entry
expect(librarian).toBeDefined()
expect(librarian.fallbackChain).toBeArray()
expect(librarian.fallbackChain.length).toBeGreaterThan(0)
const primary = librarian.fallbackChain[0]
expect(primary.providers[0]).toBe("google")
expect(primary.model).toBe("gemini-3-flash")
})
test("explore has valid fallbackChain with grok-code-fast-1 as primary", () => {
// given - explore agent requirement
const explore = AGENT_MODEL_REQUIREMENTS["explore"]
// when - accessing explore requirement
// then - fallbackChain: grok → minimax-free → haiku → nano
expect(explore).toBeDefined()
expect(explore.fallbackChain).toBeArray()
expect(explore.fallbackChain).toHaveLength(4)
const primary = explore.fallbackChain[0]
expect(primary.providers).toContain("github-copilot")
expect(primary.model).toBe("grok-code-fast-1")
const secondary = explore.fallbackChain[1]
expect(secondary.providers).toContain("opencode")
expect(secondary.model).toBe("minimax-m2.5-free")
const tertiary = explore.fallbackChain[2]
expect(tertiary.providers).toContain("anthropic")
expect(tertiary.model).toBe("claude-haiku-4-5")
const quaternary = explore.fallbackChain[3]
expect(quaternary.providers).toContain("opencode")
expect(quaternary.model).toBe("gpt-5-nano")
})
test("multimodal-looker has valid fallbackChain with k2p5 as primary", () => {
// given - multimodal-looker agent requirement
const multimodalLooker = AGENT_MODEL_REQUIREMENTS["multimodal-looker"]
// when - accessing multimodal-looker requirement
// then - fallbackChain exists with k2p5 as first entry
expect(multimodalLooker).toBeDefined()
expect(multimodalLooker.fallbackChain).toBeArray()
expect(multimodalLooker.fallbackChain.length).toBeGreaterThan(0)
const primary = multimodalLooker.fallbackChain[0]
expect(primary.providers[0]).toBe("kimi-for-coding")
expect(primary.model).toBe("k2p5")
})
test("prometheus has claude-opus-4-6 as primary", () => {
// #given - prometheus agent requirement
const prometheus = AGENT_MODEL_REQUIREMENTS["prometheus"]
// #when - accessing Prometheus requirement
// #then - claude-opus-4-6 is first
expect(prometheus).toBeDefined()
expect(prometheus.fallbackChain).toBeArray()
expect(prometheus.fallbackChain.length).toBeGreaterThan(1)
const primary = prometheus.fallbackChain[0]
expect(primary.model).toBe("claude-opus-4-6")
expect(primary.providers).toEqual(["anthropic", "github-copilot", "opencode"])
expect(primary.variant).toBe("max")
})
test("metis has claude-opus-4-6 as primary", () => {
// #given - metis agent requirement
const metis = AGENT_MODEL_REQUIREMENTS["metis"]
// #when - accessing Metis requirement
// #then - claude-opus-4-6 is first
expect(metis).toBeDefined()
expect(metis.fallbackChain).toBeArray()
expect(metis.fallbackChain.length).toBeGreaterThan(1)
const primary = metis.fallbackChain[0]
expect(primary.model).toBe("claude-opus-4-6")
expect(primary.providers).toEqual(["anthropic", "github-copilot", "opencode"])
expect(primary.variant).toBe("max")
})
test("momus has valid fallbackChain with gpt-5.2 as primary", () => {
// given - momus agent requirement
const momus = AGENT_MODEL_REQUIREMENTS["momus"]
// when - accessing Momus requirement
// then - fallbackChain exists with gpt-5.2 as first entry, variant medium
expect(momus).toBeDefined()
expect(momus.fallbackChain).toBeArray()
expect(momus.fallbackChain.length).toBeGreaterThan(0)
const primary = momus.fallbackChain[0]
expect(primary.model).toBe("gpt-5.2")
expect(primary.variant).toBe("medium")
expect(primary.providers[0]).toBe("openai")
})
test("atlas has valid fallbackChain with k2p5 as primary (kimi-for-coding prioritized)", () => {
// given - atlas agent requirement
const atlas = AGENT_MODEL_REQUIREMENTS["atlas"]
// when - accessing Atlas requirement
// then - fallbackChain exists with k2p5 as first entry (kimi-for-coding prioritized)
expect(atlas).toBeDefined()
expect(atlas.fallbackChain).toBeArray()
expect(atlas.fallbackChain.length).toBeGreaterThan(0)
const primary = atlas.fallbackChain[0]
expect(primary.model).toBe("k2p5")
expect(primary.providers[0]).toBe("kimi-for-coding")
})
test("hephaestus requires openai/github-copilot/opencode provider", () => {
// #given - hephaestus agent requirement
const hephaestus = AGENT_MODEL_REQUIREMENTS["hephaestus"]
// #when - accessing hephaestus requirement
// #then - requiresProvider is set to openai, github-copilot, opencode (not requiresModel)
expect(hephaestus).toBeDefined()
expect(hephaestus.requiresProvider).toEqual(["openai", "github-copilot", "opencode"])
expect(hephaestus.requiresModel).toBeUndefined()
})
test("all 10 builtin agents have valid fallbackChain arrays", () => {
// #given - list of 10 agent names
const expectedAgents = [
"sisyphus",
"hephaestus",
"oracle",
"librarian",
test("defines all 10 builtin agents", () => {
expect(Object.keys(AGENT_MODEL_REQUIREMENTS).sort()).toEqual([
"atlas",
"explore",
"multimodal-looker",
"prometheus",
"hephaestus",
"librarian",
"metis",
"momus",
"atlas",
]
"multimodal-looker",
"oracle",
"prometheus",
"sisyphus",
])
})
// when - checking AGENT_MODEL_REQUIREMENTS
const definedAgents = Object.keys(AGENT_MODEL_REQUIREMENTS)
test("sisyphus: 2nd fallback is quotio gpt-5.3-codex (high)", () => {
const sisyphus = AGENT_MODEL_REQUIREMENTS["sisyphus"]
expect(sisyphus.requiresAnyModel).toBe(true)
expect(sisyphus.fallbackChain.length).toBeGreaterThan(2)
// #then - all agents present with valid fallbackChain
expect(definedAgents).toHaveLength(10)
for (const agent of expectedAgents) {
const requirement = AGENT_MODEL_REQUIREMENTS[agent]
expect(requirement).toBeDefined()
expect(requirement.fallbackChain).toBeArray()
expect(requirement.fallbackChain.length).toBeGreaterThan(0)
expect(sisyphus.fallbackChain[0]).toEqual({
providers: ["quotio"],
model: "claude-opus-4-6",
variant: "max",
})
for (const entry of requirement.fallbackChain) {
expect(entry.providers).toBeArray()
expect(entry.providers.length).toBeGreaterThan(0)
expect(typeof entry.model).toBe("string")
expect(entry.model.length).toBeGreaterThan(0)
}
}
expect(sisyphus.fallbackChain[1]).toEqual({
providers: ["quotio"],
model: "gpt-5.3-codex",
variant: "high",
})
})
test("explore: uses speed chain, includes rome, and gpt-5-mini is copilot-first", () => {
const explore = AGENT_MODEL_REQUIREMENTS["explore"]
expect(explore.fallbackChain.length).toBeGreaterThan(4)
expect(explore.fallbackChain[0].model).toBe("claude-haiku-4-5")
expect(explore.fallbackChain.some((e) => e.model === "iflow-rome-30ba3b")).toBe(true)
const gptMini = explore.fallbackChain.find((e) => e.model === "gpt-5-mini")
expect(gptMini).toBeDefined()
expect(gptMini!.providers[0]).toBe("github-copilot")
expect(gptMini!.variant).toBe("high")
})
test("multimodal-looker: prefers gemini image model first", () => {
const multimodal = AGENT_MODEL_REQUIREMENTS["multimodal-looker"]
expect(multimodal.fallbackChain[0]).toEqual({
providers: ["quotio"],
model: "gemini-3-pro-image",
})
})
test("includes NVIDIA NIM additions in at least one agent chain", () => {
const all = Object.values(AGENT_MODEL_REQUIREMENTS).flatMap((r) => r.fallbackChain)
expect(all.some((e) => e.providers.includes("nvidia") && e.model === "qwen/qwen3.5-397b-a17b")).toBe(true)
expect(all.some((e) => e.providers.includes("nvidia") && e.model === "stepfun-ai/step-3.5-flash")).toBe(true)
expect(all.some((e) => e.providers.includes("nvidia") && e.model === "bytedance/seed-oss-36b-instruct")).toBe(true)
})
})
describe("CATEGORY_MODEL_REQUIREMENTS", () => {
test("ultrabrain has valid fallbackChain with gpt-5.3-codex as primary", () => {
// given - ultrabrain category requirement
const ultrabrain = CATEGORY_MODEL_REQUIREMENTS["ultrabrain"]
// when - accessing ultrabrain requirement
// then - fallbackChain exists with gpt-5.3-codex as first entry
expect(ultrabrain).toBeDefined()
expect(ultrabrain.fallbackChain).toBeArray()
expect(ultrabrain.fallbackChain.length).toBeGreaterThan(0)
const primary = ultrabrain.fallbackChain[0]
expect(primary.variant).toBe("xhigh")
expect(primary.model).toBe("gpt-5.3-codex")
expect(primary.providers[0]).toBe("openai")
})
test("deep has valid fallbackChain with gpt-5.3-codex as primary", () => {
// given - deep category requirement
const deep = CATEGORY_MODEL_REQUIREMENTS["deep"]
// when - accessing deep requirement
// then - fallbackChain exists with gpt-5.3-codex as first entry, medium variant
expect(deep).toBeDefined()
expect(deep.fallbackChain).toBeArray()
expect(deep.fallbackChain.length).toBeGreaterThan(0)
const primary = deep.fallbackChain[0]
expect(primary.variant).toBe("medium")
expect(primary.model).toBe("gpt-5.3-codex")
expect(primary.providers[0]).toBe("openai")
})
test("visual-engineering has valid fallbackChain with gemini-3-pro high as primary", () => {
// given - visual-engineering category requirement
const visualEngineering = CATEGORY_MODEL_REQUIREMENTS["visual-engineering"]
// when - accessing visual-engineering requirement
// then - fallbackChain: gemini-3-pro(high) → glm-5 → opus-4-6(max) → k2p5
expect(visualEngineering).toBeDefined()
expect(visualEngineering.fallbackChain).toBeArray()
expect(visualEngineering.fallbackChain).toHaveLength(4)
const primary = visualEngineering.fallbackChain[0]
expect(primary.providers[0]).toBe("google")
expect(primary.model).toBe("gemini-3-pro")
expect(primary.variant).toBe("high")
const second = visualEngineering.fallbackChain[1]
expect(second.providers[0]).toBe("zai-coding-plan")
expect(second.model).toBe("glm-5")
const third = visualEngineering.fallbackChain[2]
expect(third.model).toBe("claude-opus-4-6")
expect(third.variant).toBe("max")
const fourth = visualEngineering.fallbackChain[3]
expect(fourth.providers[0]).toBe("kimi-for-coding")
expect(fourth.model).toBe("k2p5")
})
test("quick has valid fallbackChain with claude-haiku-4-5 as primary", () => {
// given - quick category requirement
const quick = CATEGORY_MODEL_REQUIREMENTS["quick"]
// when - accessing quick requirement
// then - fallbackChain exists with claude-haiku-4-5 as first entry
expect(quick).toBeDefined()
expect(quick.fallbackChain).toBeArray()
expect(quick.fallbackChain.length).toBeGreaterThan(0)
const primary = quick.fallbackChain[0]
expect(primary.model).toBe("claude-haiku-4-5")
expect(primary.providers[0]).toBe("anthropic")
})
test("unspecified-low has valid fallbackChain with claude-sonnet-4-6 as primary", () => {
// given - unspecified-low category requirement
const unspecifiedLow = CATEGORY_MODEL_REQUIREMENTS["unspecified-low"]
// when - accessing unspecified-low requirement
// then - fallbackChain exists with claude-sonnet-4-6 as first entry
expect(unspecifiedLow).toBeDefined()
expect(unspecifiedLow.fallbackChain).toBeArray()
expect(unspecifiedLow.fallbackChain.length).toBeGreaterThan(0)
const primary = unspecifiedLow.fallbackChain[0]
expect(primary.model).toBe("claude-sonnet-4-6")
expect(primary.providers[0]).toBe("anthropic")
})
test("unspecified-high has claude-opus-4-6 as primary", () => {
// #given - unspecified-high category requirement
const unspecifiedHigh = CATEGORY_MODEL_REQUIREMENTS["unspecified-high"]
// #when - accessing unspecified-high requirement
// #then - claude-opus-4-6 is first
expect(unspecifiedHigh).toBeDefined()
expect(unspecifiedHigh.fallbackChain).toBeArray()
expect(unspecifiedHigh.fallbackChain.length).toBeGreaterThan(1)
const primary = unspecifiedHigh.fallbackChain[0]
expect(primary.model).toBe("claude-opus-4-6")
expect(primary.variant).toBe("max")
expect(primary.providers).toEqual(["anthropic", "github-copilot", "opencode"])
})
test("artistry has valid fallbackChain with gemini-3-pro as primary", () => {
// given - artistry category requirement
const artistry = CATEGORY_MODEL_REQUIREMENTS["artistry"]
// when - accessing artistry requirement
// then - fallbackChain exists with gemini-3-pro as first entry
expect(artistry).toBeDefined()
expect(artistry.fallbackChain).toBeArray()
expect(artistry.fallbackChain.length).toBeGreaterThan(0)
const primary = artistry.fallbackChain[0]
expect(primary.model).toBe("gemini-3-pro")
expect(primary.variant).toBe("high")
expect(primary.providers[0]).toBe("google")
})
test("writing has valid fallbackChain with k2p5 as primary (kimi-for-coding)", () => {
// given - writing category requirement
const writing = CATEGORY_MODEL_REQUIREMENTS["writing"]
// when - accessing writing requirement
// then - fallbackChain: k2p5 → gemini-3-flash → claude-sonnet-4-6
expect(writing).toBeDefined()
expect(writing.fallbackChain).toBeArray()
expect(writing.fallbackChain).toHaveLength(3)
const primary = writing.fallbackChain[0]
expect(primary.model).toBe("k2p5")
expect(primary.providers[0]).toBe("kimi-for-coding")
const second = writing.fallbackChain[1]
expect(second.model).toBe("gemini-3-flash")
expect(second.providers[0]).toBe("google")
})
test("all 8 categories have valid fallbackChain arrays", () => {
// given - list of 8 category names
const expectedCategories = [
"visual-engineering",
"ultrabrain",
"deep",
test("defines all 8 categories", () => {
expect(Object.keys(CATEGORY_MODEL_REQUIREMENTS).sort()).toEqual([
"artistry",
"deep",
"quick",
"unspecified-low",
"ultrabrain",
"unspecified-high",
"unspecified-low",
"visual-engineering",
"writing",
]
])
})
// when - checking CATEGORY_MODEL_REQUIREMENTS
const definedCategories = Object.keys(CATEGORY_MODEL_REQUIREMENTS)
test("deep requires gpt-5.3-codex", () => {
expect(CATEGORY_MODEL_REQUIREMENTS["deep"].requiresModel).toBe("gpt-5.3-codex")
})
// then - all categories present with valid fallbackChain
expect(definedCategories).toHaveLength(8)
for (const category of expectedCategories) {
const requirement = CATEGORY_MODEL_REQUIREMENTS[category]
expect(requirement).toBeDefined()
expect(requirement.fallbackChain).toBeArray()
expect(requirement.fallbackChain.length).toBeGreaterThan(0)
test("quick uses the speed chain (haiku primary)", () => {
expect(CATEGORY_MODEL_REQUIREMENTS["quick"].fallbackChain[0].model).toBe("claude-haiku-4-5")
})
for (const entry of requirement.fallbackChain) {
expect(entry.providers).toBeArray()
expect(entry.providers.length).toBeGreaterThan(0)
expect(typeof entry.model).toBe("string")
expect(entry.model.length).toBeGreaterThan(0)
}
test("ultrabrain starts with gpt-5.3-codex (high)", () => {
const ultrabrain = CATEGORY_MODEL_REQUIREMENTS["ultrabrain"]
expect(ultrabrain.fallbackChain[0]).toEqual({
providers: ["quotio"],
model: "gpt-5.3-codex",
variant: "high",
})
})
})
describe("ModelRequirements invariants", () => {
test("all entries have non-empty providers and a non-empty model", () => {
for (const entry of flattenChains()) {
expect(entry.providers.length).toBeGreaterThan(0)
expect(typeof entry.model).toBe("string")
expect(entry.model.length).toBeGreaterThan(0)
}
})
test("no entry uses opencode provider and no excluded models are present", () => {
for (const entry of flattenChains()) {
assertNoOpencodeProvider(entry)
assertNoExcludedModels(entry)
assertNoProviderPrefixForNonNamespacedProviders(entry)
}
})
})
describe("FallbackEntry type", () => {
test("FallbackEntry structure is correct", () => {
// given - a valid FallbackEntry object
const entry: FallbackEntry = {
providers: ["anthropic", "github-copilot", "opencode"],
model: "claude-opus-4-6",
variant: "high",
}
// when - accessing properties
// then - all properties are accessible
expect(entry.providers).toEqual(["anthropic", "github-copilot", "opencode"])
expect(entry.model).toBe("claude-opus-4-6")
expect(entry.variant).toBe("high")
})
test("FallbackEntry variant is optional", () => {
// given - a FallbackEntry without variant
const entry: FallbackEntry = {
providers: ["opencode", "anthropic"],
model: "big-pickle",
}
// when - accessing variant
// then - variant is undefined
describe("Type sanity", () => {
test("FallbackEntry.variant is optional", () => {
const entry: FallbackEntry = { providers: ["quotio"], model: "claude-haiku-4-5" }
expect(entry.variant).toBeUndefined()
})
})
describe("ModelRequirement type", () => {
test("ModelRequirement structure with fallbackChain is correct", () => {
// given - a valid ModelRequirement object
const requirement: ModelRequirement = {
fallbackChain: [
{ providers: ["anthropic", "github-copilot"], model: "claude-opus-4-6", variant: "max" },
{ providers: ["openai", "github-copilot"], model: "gpt-5.2", variant: "high" },
],
}
// when - accessing properties
// then - fallbackChain is accessible with correct structure
expect(requirement.fallbackChain).toBeArray()
expect(requirement.fallbackChain).toHaveLength(2)
expect(requirement.fallbackChain[0].model).toBe("claude-opus-4-6")
expect(requirement.fallbackChain[1].model).toBe("gpt-5.2")
})
test("ModelRequirement variant is optional", () => {
// given - a ModelRequirement without top-level variant
const requirement: ModelRequirement = {
fallbackChain: [{ providers: ["opencode"], model: "big-pickle" }],
}
// when - accessing variant
// then - variant is undefined
expect(requirement.variant).toBeUndefined()
})
test("no model in fallbackChain has provider prefix", () => {
// given - all agent and category requirements
const allRequirements = [
...Object.values(AGENT_MODEL_REQUIREMENTS),
...Object.values(CATEGORY_MODEL_REQUIREMENTS),
]
// when - checking each model in fallbackChain
// then - none contain "/" (provider prefix)
for (const req of allRequirements) {
for (const entry of req.fallbackChain) {
expect(entry.model).not.toContain("/")
}
}
})
test("all fallbackChain entries have non-empty providers array", () => {
// given - all agent and category requirements
const allRequirements = [
...Object.values(AGENT_MODEL_REQUIREMENTS),
...Object.values(CATEGORY_MODEL_REQUIREMENTS),
]
// when - checking each entry in fallbackChain
// then - all have non-empty providers array
for (const req of allRequirements) {
for (const entry of req.fallbackChain) {
expect(entry.providers).toBeArray()
expect(entry.providers.length).toBeGreaterThan(0)
}
}
})
})
describe("requiresModel field in categories", () => {
test("deep category has requiresModel set to gpt-5.3-codex", () => {
// given
const deep = CATEGORY_MODEL_REQUIREMENTS["deep"]
// when / #then
expect(deep.requiresModel).toBe("gpt-5.3-codex")
})
test("artistry category has requiresModel set to gemini-3-pro", () => {
// given
const artistry = CATEGORY_MODEL_REQUIREMENTS["artistry"]
// when / #then
expect(artistry.requiresModel).toBe("gemini-3-pro")
test("ModelRequirement.variant is optional", () => {
const req: ModelRequirement = { fallbackChain: [{ providers: ["quotio"], model: "claude-haiku-4-5" }] }
expect(req.variant).toBeUndefined()
})
})
+121 -74
View File
@@ -12,85 +12,133 @@ export type ModelRequirement = {
requiresProvider?: string[] // If set, only activates when any of these providers is connected
}
function fb(providers: string[] | string, model: string, variant?: string): FallbackEntry {
return {
providers: Array.isArray(providers) ? providers : [providers],
model,
...(variant !== undefined ? { variant } : {}),
}
}
// Provider preference rules:
// - Never use the paid `opencode` provider as an automatic fallback.
// - Prefer `quotio` when the same model exists across multiple providers.
// - Prefer `github-copilot` first for `gpt-5-mini` (unlimited), fall back to `quotio`.
// Note: user requested "Quotio-first" and to avoid the OpenCode provider; we keep runtime fallbacks on
// `quotio` + `nvidia` (+ `github-copilot` for unlimited GPT mini) unless explicitly requested otherwise.
const P_GPT: string[] = ["quotio"]
const P_GPT_MINI: string[] = ["github-copilot", "quotio"]
// Benchmark-driven ordering (user-provided table + NVIDIA NIM docs), tuned per-agent for quality vs speed.
const SPEED_CHAIN: FallbackEntry[] = [
fb("quotio", "claude-haiku-4-5"), fb("quotio", "oswe-vscode-prime"),
fb(P_GPT_MINI, "gpt-5-mini", "high"), fb(P_GPT_MINI, "gpt-4.1"),
fb("nvidia", "nvidia/nemotron-3-nano-30b-a3b"), fb("quotio", "iflow-rome-30ba3b"),
fb("minimax-coding-plan", "MiniMax-M2.5"), fb("nvidia", "bytedance/seed-oss-36b-instruct"),
fb("quotio", "claude-sonnet-4-5"),
]
const QUALITY_CODING_CHAIN: FallbackEntry[] = [
fb("quotio", "claude-opus-4-6-thinking"),
fb("nvidia", "stepfun-ai/step-3.5-flash"),
fb("nvidia", "qwen/qwen3.5-397b-a17b"),
fb("quotio", "glm-5"),
fb("nvidia", "z-ai/glm5"),
fb("quotio", "deepseek-v3.2-reasoner"),
fb("quotio", "deepseek-r1"),
fb("nvidia", "deepseek-ai/deepseek-r1"),
fb("quotio", "qwen3-235b-a22b-thinking-2507"),
fb("nvidia", "qwen/qwen3-next-80b-a3b-thinking"),
fb("nvidia", "qwen/qwen3-coder-480b-a35b-instruct"),
fb("nvidia", "bytedance/seed-oss-36b-instruct"),
fb("quotio", "kimi-k2-thinking"),
fb("quotio", "kimi-k2.5"),
fb("nvidia", "moonshotai/kimi-k2.5"),
fb("minimax-coding-plan", "MiniMax-M2.5"),
fb("minimax-coding-plan", "MiniMax-M2.5-highspeed"),
fb("minimax", "MiniMax-M2.5"),
fb("quotio", "minimax-m2.5"),
fb("quotio", "claude-sonnet-4-5-thinking"),
]
export const AGENT_MODEL_REQUIREMENTS: Record<string, ModelRequirement> = {
sisyphus: {
fallbackChain: [
{ providers: ["anthropic", "github-copilot", "opencode"], model: "claude-opus-4-6", variant: "max" },
{ providers: ["kimi-for-coding"], model: "k2p5" },
{ providers: ["opencode"], model: "kimi-k2.5-free" },
{ providers: ["zai-coding-plan", "opencode"], model: "glm-5" },
{ providers: ["opencode"], model: "big-pickle" },
// 1st fallback: switch away from Opus Thinking to the non-thinking model (often more available).
fb("quotio", "claude-opus-4-6", "max"),
// 2nd fallback: user-requested.
fb("quotio", "gpt-5.3-codex", "high"),
...QUALITY_CODING_CHAIN,
...SPEED_CHAIN,
],
requiresAnyModel: true,
},
hephaestus: {
fallbackChain: [
{ providers: ["openai", "github-copilot", "opencode"], model: "gpt-5.3-codex", variant: "medium" },
fb("quotio", "gpt-5.3-codex", "high"),
...QUALITY_CODING_CHAIN,
],
requiresProvider: ["openai", "github-copilot", "opencode"],
requiresAnyModel: true,
},
oracle: {
fallbackChain: [
{ providers: ["openai", "github-copilot", "opencode"], model: "gpt-5.2", variant: "high" },
{ providers: ["google", "github-copilot", "opencode"], model: "gemini-3-pro", variant: "high" },
{ providers: ["anthropic", "github-copilot", "opencode"], model: "claude-opus-4-6", variant: "max" },
fb("quotio", "gpt-5.3-codex", "high"),
fb("quotio", "claude-opus-4-6-thinking"),
fb("quotio", "claude-sonnet-4-5-thinking"),
...QUALITY_CODING_CHAIN,
],
},
librarian: {
fallbackChain: [
{ providers: ["google", "github-copilot", "opencode"], model: "gemini-3-flash" },
{ providers: ["opencode"], model: "minimax-m2.5-free" },
{ providers: ["opencode"], model: "big-pickle" },
],
},
explore: {
librarian: {
fallbackChain: [
{ providers: ["github-copilot"], model: "grok-code-fast-1" },
{ providers: ["opencode"], model: "minimax-m2.5-free" },
{ providers: ["anthropic", "opencode"], model: "claude-haiku-4-5" },
{ providers: ["opencode"], model: "gpt-5-nano" },
fb("quotio", "claude-sonnet-4-5"),
...SPEED_CHAIN,
...QUALITY_CODING_CHAIN,
],
},
explore: {
fallbackChain: SPEED_CHAIN,
},
"multimodal-looker": {
fallbackChain: [
{ providers: ["kimi-for-coding"], model: "k2p5" },
{ providers: ["opencode"], model: "kimi-k2.5-free" },
{ providers: ["google", "github-copilot", "opencode"], model: "gemini-3-flash" },
{ providers: ["openai", "github-copilot", "opencode"], model: "gpt-5.2" },
{ providers: ["zai-coding-plan"], model: "glm-4.6v" },
fb("quotio", "gemini-3-pro-image"),
fb("quotio", "gemini-3-pro-high"),
fb("quotio", "gemini-3-flash"),
fb("quotio", "kimi-k2.5"),
fb("quotio", "claude-opus-4-6-thinking"),
fb("quotio", "claude-sonnet-4-5-thinking"),
fb("quotio", "claude-haiku-4-5"),
],
},
prometheus: {
fallbackChain: [
{ providers: ["anthropic", "github-copilot", "opencode"], model: "claude-opus-4-6", variant: "max" },
{ providers: ["openai", "github-copilot", "opencode"], model: "gpt-5.2", variant: "high" },
{ providers: ["kimi-for-coding"], model: "k2p5" },
{ providers: ["opencode"], model: "kimi-k2.5-free" },
{ providers: ["google", "github-copilot", "opencode"], model: "gemini-3-pro" },
fb("quotio", "claude-opus-4-6-thinking"),
fb("quotio", "gpt-5.3-codex", "high"),
fb("quotio", "claude-sonnet-4-5-thinking"),
...QUALITY_CODING_CHAIN,
],
},
metis: {
fallbackChain: [
{ providers: ["anthropic", "github-copilot", "opencode"], model: "claude-opus-4-6", variant: "max" },
{ providers: ["kimi-for-coding"], model: "k2p5" },
{ providers: ["opencode"], model: "kimi-k2.5-free" },
{ providers: ["openai", "github-copilot", "opencode"], model: "gpt-5.2", variant: "high" },
{ providers: ["google", "github-copilot", "opencode"], model: "gemini-3-pro", variant: "high" },
fb("quotio", "claude-opus-4-6-thinking"),
fb("quotio", "gpt-5.3-codex", "high"),
fb("quotio", "claude-sonnet-4-5-thinking"),
...QUALITY_CODING_CHAIN,
],
},
momus: {
fallbackChain: [
{ providers: ["openai", "github-copilot", "opencode"], model: "gpt-5.2", variant: "medium" },
{ providers: ["anthropic", "github-copilot", "opencode"], model: "claude-opus-4-6", variant: "max" },
{ providers: ["google", "github-copilot", "opencode"], model: "gemini-3-pro", variant: "high" },
fb("quotio", "gpt-5.3-codex", "high"),
fb("quotio", "claude-opus-4-6-thinking"),
...QUALITY_CODING_CHAIN,
],
},
atlas: {
fallbackChain: [
{ providers: ["kimi-for-coding"], model: "k2p5" },
{ providers: ["opencode"], model: "kimi-k2.5-free" },
{ providers: ["anthropic", "github-copilot", "opencode"], model: "claude-sonnet-4-6" },
{ providers: ["openai", "github-copilot", "opencode"], model: "gpt-5.2" },
fb("quotio", "claude-sonnet-4-5-thinking"),
fb("quotio", "claude-opus-4-6-thinking"),
fb("quotio", "gpt-5.3-codex", "medium"),
...QUALITY_CODING_CHAIN,
],
},
}
@@ -98,61 +146,60 @@ export const AGENT_MODEL_REQUIREMENTS: Record<string, ModelRequirement> = {
export const CATEGORY_MODEL_REQUIREMENTS: Record<string, ModelRequirement> = {
"visual-engineering": {
fallbackChain: [
{ providers: ["google", "github-copilot", "opencode"], model: "gemini-3-pro", variant: "high" },
{ providers: ["zai-coding-plan", "opencode"], model: "glm-5" },
{ providers: ["anthropic", "github-copilot", "opencode"], model: "claude-opus-4-6", variant: "max" },
{ providers: ["kimi-for-coding"], model: "k2p5" },
fb("quotio", "claude-opus-4-6-thinking"),
fb("quotio", "gemini-3-pro-image"),
fb("quotio", "kimi-k2-thinking"),
fb("quotio", "kimi-k2.5"),
fb("quotio", "claude-sonnet-4-5-thinking"),
fb("quotio", "gpt-5.3-codex", "medium"),
],
},
ultrabrain: {
fallbackChain: [
{ providers: ["openai", "github-copilot", "opencode"], model: "gpt-5.3-codex", variant: "xhigh" },
{ providers: ["google", "github-copilot", "opencode"], model: "gemini-3-pro", variant: "high" },
{ providers: ["anthropic", "github-copilot", "opencode"], model: "claude-opus-4-6", variant: "max" },
fb("quotio", "gpt-5.3-codex", "high"),
fb("quotio", "claude-opus-4-6-thinking"),
fb("nvidia", "stepfun-ai/step-3.5-flash"),
fb("nvidia", "qwen/qwen3.5-397b-a17b"),
...QUALITY_CODING_CHAIN,
],
},
deep: {
fallbackChain: [
{ providers: ["openai", "github-copilot", "opencode"], model: "gpt-5.3-codex", variant: "medium" },
{ providers: ["anthropic", "github-copilot", "opencode"], model: "claude-opus-4-6", variant: "max" },
{ providers: ["google", "github-copilot", "opencode"], model: "gemini-3-pro", variant: "high" },
fb("quotio", "gpt-5.3-codex", "medium"),
fb("quotio", "claude-opus-4-6-thinking"),
fb("quotio", "claude-sonnet-4-5-thinking"),
...QUALITY_CODING_CHAIN,
],
requiresModel: "gpt-5.3-codex",
},
artistry: {
fallbackChain: [
{ providers: ["google", "github-copilot", "opencode"], model: "gemini-3-pro", variant: "high" },
{ providers: ["anthropic", "github-copilot", "opencode"], model: "claude-opus-4-6", variant: "max" },
{ providers: ["openai", "github-copilot", "opencode"], model: "gpt-5.2" },
fb("quotio", "claude-opus-4-6-thinking"),
fb("quotio", "claude-sonnet-4-5-thinking"),
fb("quotio", "claude-sonnet-4-5"),
],
requiresModel: "gemini-3-pro",
requiresModel: "claude-opus-4-6",
},
quick: {
fallbackChain: [
{ providers: ["anthropic", "github-copilot", "opencode"], model: "claude-haiku-4-5" },
{ providers: ["google", "github-copilot", "opencode"], model: "gemini-3-flash" },
{ providers: ["opencode"], model: "gpt-5-nano" },
],
fallbackChain: SPEED_CHAIN,
},
"unspecified-low": {
fallbackChain: [
{ providers: ["anthropic", "github-copilot", "opencode"], model: "claude-sonnet-4-6" },
{ providers: ["openai", "github-copilot", "opencode"], model: "gpt-5.3-codex", variant: "medium" },
{ providers: ["google", "github-copilot", "opencode"], model: "gemini-3-flash" },
],
fallbackChain: SPEED_CHAIN,
},
"unspecified-high": {
fallbackChain: [
{ providers: ["anthropic", "github-copilot", "opencode"], model: "claude-opus-4-6", variant: "max" },
{ providers: ["openai", "github-copilot", "opencode"], model: "gpt-5.2", variant: "high" },
{ providers: ["google", "github-copilot", "opencode"], model: "gemini-3-pro" },
fb("quotio", "claude-opus-4-6-thinking"),
fb("quotio", "gpt-5.3-codex", "high"),
...QUALITY_CODING_CHAIN,
],
},
writing: {
fallbackChain: [
{ providers: ["kimi-for-coding"], model: "k2p5" },
{ providers: ["google", "github-copilot", "opencode"], model: "gemini-3-flash" },
{ providers: ["anthropic", "github-copilot", "opencode"], model: "claude-sonnet-4-6" },
fb("quotio", "claude-sonnet-4-5"),
fb("quotio", "glm-5"),
fb("quotio", "kimi-k2.5"),
fb("quotio", "claude-haiku-4-5"),
fb("quotio", "gemini-3-flash"),
],
},
}
+30
View File
@@ -0,0 +1,30 @@
import { describe, expect, test } from "bun:test"
import { clearSessionModel, getSessionModel, setSessionModel } from "./session-model-state"
describe("session-model-state", () => {
test("stores and retrieves a session model", () => {
//#given
const sessionID = "ses_test"
//#when
setSessionModel(sessionID, { providerID: "github-copilot", modelID: "gpt-4.1" })
//#then
expect(getSessionModel(sessionID)).toEqual({
providerID: "github-copilot",
modelID: "gpt-4.1",
})
})
test("clears a session model", () => {
//#given
const sessionID = "ses_clear"
setSessionModel(sessionID, { providerID: "quotio", modelID: "gpt-5.3-codex" })
//#when
clearSessionModel(sessionID)
//#then
expect(getSessionModel(sessionID)).toBeUndefined()
})
})
+15
View File
@@ -0,0 +1,15 @@
export type SessionModel = { providerID: string; modelID: string }
const sessionModels = new Map<string, SessionModel>()
export function setSessionModel(sessionID: string, model: SessionModel): void {
sessionModels.set(sessionID, model)
}
export function getSessionModel(sessionID: string): SessionModel | undefined {
return sessionModels.get(sessionID)
}
export function clearSessionModel(sessionID: string): void {
sessionModels.delete(sessionID)
}