update default model setup priorities

This commit is contained in:
YeonGyu-Kim
2026-05-08 12:18:38 +09:00
parent cd31d2a1a8
commit 12c4795e1b
14 changed files with 149 additions and 117 deletions
+25 -23
View File
@@ -199,18 +199,18 @@ describe("AGENT_MODEL_REQUIREMENTS", () => {
})
})
test("momus has valid fallbackChain with gpt-5.5 as primary", () => {
test("momus has valid fallbackChain with gpt-5.2 as primary", () => {
// given - momus agent requirement
const momus = AGENT_MODEL_REQUIREMENTS["momus"]
// when - accessing Momus requirement
// then - fallbackChain exists with gpt-5.5 as first entry, variant xhigh
// then - fallbackChain exists with gpt-5.2 as first entry, variant xhigh
expect(momus).toBeDefined()
expect(momus.fallbackChain).toBeArray()
expect(momus.fallbackChain.length).toBeGreaterThan(0)
const primary = momus.fallbackChain[0]
expect(primary.model).toBe("gpt-5.5")
expect(primary.model).toBe("gpt-5.2")
expect(primary.variant).toBe("xhigh")
expect(primary.providers[0]).toBe("openai")
})
@@ -331,21 +331,21 @@ describe("CATEGORY_MODEL_REQUIREMENTS", () => {
expect(primary.providers[0]).toBe("openai")
})
test("deep has valid fallbackChain with gpt-5.5 as primary", () => {
test("deep has valid fallbackChain with gpt-5.3-codex as primary", () => {
// given - deep category requirement
const deep = CATEGORY_MODEL_REQUIREMENTS["deep"]
// when - accessing deep requirement
// then - fallbackChain exists with gpt-5.5 as first entry, medium variant
// then - fallbackChain exists with gpt-5.3-codex as first entry, high variant
expect(deep).toBeDefined()
expect(deep.fallbackChain).toBeArray()
expect(deep.fallbackChain.length).toBeGreaterThan(0)
const primary = deep.fallbackChain[0]
expect(primary.variant).toBe("medium")
expect(primary.model).toBe("gpt-5.5")
expect(primary.variant).toBe("high")
expect(primary.model).toBe("gpt-5.3-codex")
expect(primary.providers).toContain("openai")
expect(primary.providers).toContain("github-copilot")
expect(primary.providers).not.toContain("github-copilot")
})
test("visual-engineering has valid fallbackChain with gemini-3.1-pro high as primary", () => {
@@ -380,18 +380,19 @@ describe("CATEGORY_MODEL_REQUIREMENTS", () => {
expect(fifth.model).toBe("k2p5")
})
test("quick has valid fallbackChain with gpt-5.4-mini as primary and claude-haiku-4-5 as secondary", () => {
test("quick has valid fallbackChain with gpt-5.4-mini-fast none as primary and claude-haiku-4-5 as secondary", () => {
// given - quick category requirement
const quick = CATEGORY_MODEL_REQUIREMENTS["quick"]
// when - accessing quick requirement
// then - fallbackChain exists with gpt-5.4-mini as first entry, haiku as second
// then - fallbackChain exists with gpt-5.4-mini-fast none as first entry, haiku as second
expect(quick).toBeDefined()
expect(quick.fallbackChain).toBeArray()
expect(quick.fallbackChain.length).toBeGreaterThan(1)
const primary = quick.fallbackChain[0]
expect(primary.model).toBe("gpt-5.4-mini")
expect(primary.model).toBe("gpt-5.4-mini-fast")
expect(primary.variant).toBe("none")
expect(primary.providers).toContain("openai")
const secondary = quick.fallbackChain[1]
@@ -399,40 +400,41 @@ describe("CATEGORY_MODEL_REQUIREMENTS", () => {
expect(secondary.providers).toContain("anthropic")
})
test("unspecified-low has valid fallbackChain with claude-sonnet-4-6 as primary", () => {
test("unspecified-low has valid fallbackChain with gpt-5.5 medium as primary", () => {
// given - unspecified-low category requirement
const unspecifiedLow = CATEGORY_MODEL_REQUIREMENTS["unspecified-low"]
// when - accessing unspecified-low requirement
// then - fallbackChain exists with claude-sonnet-4-6 as first entry
// then - fallbackChain exists with gpt-5.5 medium as first entry
expect(unspecifiedLow).toBeDefined()
expect(unspecifiedLow.fallbackChain).toBeArray()
expect(unspecifiedLow.fallbackChain.length).toBeGreaterThan(0)
const primary = unspecifiedLow.fallbackChain[0]
expect(primary.model).toBe("claude-sonnet-4-6")
expect(primary.providers[0]).toBe("anthropic")
expect(primary.model).toBe("gpt-5.5")
expect(primary.variant).toBe("medium")
expect(primary.providers[0]).toBe("openai")
})
test("unspecified-high has claude-opus-4-7 as primary and gpt-5.5 as secondary", () => {
test("unspecified-high has gpt-5.5 high as primary and claude-opus-4-7 as secondary", () => {
// #given - unspecified-high category requirement
const unspecifiedHigh = CATEGORY_MODEL_REQUIREMENTS["unspecified-high"]
// #when - accessing unspecified-high requirement
// #then - claude-opus-4-7 is first and gpt-5.5 is second
// #then - gpt-5.5 is first and claude-opus-4-7 is second
expect(unspecifiedHigh).toBeDefined()
expect(unspecifiedHigh.fallbackChain).toBeArray()
expect(unspecifiedHigh.fallbackChain.length).toBeGreaterThan(1)
const primary = unspecifiedHigh.fallbackChain[0]
expect(primary.model).toBe("claude-opus-4-7")
expect(primary.variant).toBe("max")
expect(primary.providers).toEqual(["anthropic", "github-copilot", "opencode", "vercel"])
expect(primary.model).toBe("gpt-5.5")
expect(primary.variant).toBe("high")
expect(primary.providers).toEqual(["openai", "github-copilot", "opencode", "vercel"])
const secondary = unspecifiedHigh.fallbackChain[1]
expect(secondary.model).toBe("gpt-5.5")
expect(secondary.variant).toBe("high")
expect(secondary.providers).toEqual(["openai", "github-copilot", "opencode", "vercel"])
expect(secondary.model).toBe("claude-opus-4-7")
expect(secondary.variant).toBe("max")
expect(secondary.providers).toEqual(["anthropic", "github-copilot", "opencode", "vercel"])
})
test("artistry has valid fallbackChain with gemini-3.1-pro as primary", () => {
+18 -12
View File
@@ -142,7 +142,7 @@ export const AGENT_MODEL_REQUIREMENTS: Record<string, ModelRequirement> = {
fallbackChain: [
{
providers: ["openai", "github-copilot", "opencode", "vercel"],
model: "gpt-5.5",
model: "gpt-5.2",
variant: "xhigh",
},
{
@@ -226,9 +226,9 @@ export const CATEGORY_MODEL_REQUIREMENTS: Record<string, ModelRequirement> = {
deep: {
fallbackChain: [
{
providers: ["openai", "github-copilot", "venice", "opencode", "vercel"],
model: "gpt-5.5",
variant: "medium",
providers: ["openai", "venice", "opencode", "vercel"],
model: "gpt-5.3-codex",
variant: "high",
},
{
providers: ["anthropic", "github-copilot", "opencode", "vercel"],
@@ -261,7 +261,8 @@ export const CATEGORY_MODEL_REQUIREMENTS: Record<string, ModelRequirement> = {
fallbackChain: [
{
providers: ["openai", "github-copilot", "opencode", "vercel"],
model: "gpt-5.4-mini",
model: "gpt-5.4-mini-fast",
variant: "none",
},
{
providers: ["anthropic", "github-copilot", "opencode", "vercel"],
@@ -278,14 +279,19 @@ export const CATEGORY_MODEL_REQUIREMENTS: Record<string, ModelRequirement> = {
"unspecified-low": {
fallbackChain: [
{
providers: ["anthropic", "github-copilot", "opencode", "vercel"],
model: "claude-sonnet-4-6",
providers: ["openai", "github-copilot", "opencode", "vercel"],
model: "gpt-5.5",
variant: "medium",
},
{
providers: ["openai", "opencode", "vercel"],
model: "gpt-5.3-codex",
variant: "medium",
},
{
providers: ["anthropic", "github-copilot", "opencode", "vercel"],
model: "claude-sonnet-4-6",
},
{ providers: ["opencode-go", "vercel"], model: "kimi-k2.6" },
{
providers: ["google", "github-copilot", "opencode", "vercel"],
@@ -296,16 +302,16 @@ export const CATEGORY_MODEL_REQUIREMENTS: Record<string, ModelRequirement> = {
},
"unspecified-high": {
fallbackChain: [
{
providers: ["anthropic", "github-copilot", "opencode", "vercel"],
model: "claude-opus-4-7",
variant: "max",
},
{
providers: ["openai", "github-copilot", "opencode", "vercel"],
model: "gpt-5.5",
variant: "high",
},
{
providers: ["anthropic", "github-copilot", "opencode", "vercel"],
model: "claude-opus-4-7",
variant: "max",
},
{ providers: ["zai-coding-plan", "opencode", "vercel"], model: "glm-5" },
{ providers: ["kimi-for-coding"], model: "k2p5" },
{ providers: ["opencode-go", "vercel"], model: "glm-5.1" },
@@ -15,7 +15,7 @@ This is NOT a default choice - it's for genuinely unclassifiable moderate-effort
</Category_Context>
<Caller_Warning>
THIS CATEGORY USES A MID-TIER MODEL (claude-sonnet-4-6).
THIS CATEGORY USES A MID-TIER MODEL (gpt-5.5 medium).
**PROVIDE CLEAR STRUCTURE:**
1. MUST DO: Enumerate required actions explicitly
@@ -41,13 +41,13 @@ If task is unclassifiable but moderate-effort, use unspecified-low instead.
export const ANTHROPIC_CATEGORIES: BuiltinCategoryDefinition[] = [
{
name: "unspecified-low",
config: { model: "anthropic/claude-sonnet-4-6" },
config: { model: "openai/gpt-5.5", variant: "medium" },
description: "Tasks that don't fit other categories, low effort required",
promptAppend: UNSPECIFIED_LOW_CATEGORY_PROMPT_APPEND,
},
{
name: "unspecified-high",
config: { model: "anthropic/claude-opus-4-7", variant: "max" },
config: { model: "openai/gpt-5.5", variant: "high" },
description: "Tasks that don't fit other categories, high effort required",
promptAppend: UNSPECIFIED_HIGH_CATEGORY_PROMPT_APPEND,
},
+3 -3
View File
@@ -158,7 +158,7 @@ Approach:
</Category_Context>
<Caller_Warning>
THIS CATEGORY USES A SMALLER/FASTER MODEL (gpt-5.4-mini).
THIS CATEGORY USES A SMALLER/FASTER MODEL (gpt-5.4-mini-fast with none reasoning effort).
The model executing this task is optimized for speed over depth. Your prompt MUST be:
@@ -202,14 +202,14 @@ export const OPENAI_CATEGORIES: BuiltinCategoryDefinition[] = [
},
{
name: "deep",
config: { model: "openai/gpt-5.5", variant: "medium" },
config: { model: "openai/gpt-5.3-codex", variant: "high" },
description: "Goal-oriented autonomous problem-solving on hairy problems requiring deep research. ONE goal + ONE deliverable per call — multiple goals must fan out as parallel `deep` calls, never bundled into one.",
promptAppend: DEEP_CATEGORY_PROMPT_APPEND,
resolvePromptAppend: resolveDeepCategoryPromptAppend,
},
{
name: "quick",
config: { model: "openai/gpt-5.4-mini" },
config: { model: "openai/gpt-5.4-mini-fast", variant: "none" },
description: "Trivial tasks - single file changes, typo fixes, simple modifications",
promptAppend: QUICK_CATEGORY_PROMPT_APPEND,
},
+44 -21
View File
@@ -34,6 +34,7 @@ const TEST_AVAILABLE_MODELS = new Set([
"google/gemini-3.1-pro",
"google/gemini-3-flash",
"openai/gpt-5.4-mini",
"openai/gpt-5.4-mini-fast",
"openai/gpt-5.5",
"openai/gpt-5.3-codex",
])
@@ -69,7 +70,7 @@ describe("sisyphus-task", () => {
models: {
anthropic: ["claude-opus-4-7", "claude-sonnet-4-6", "claude-haiku-4-5"],
google: ["gemini-3.1-pro", "gemini-3-flash"],
openai: ["gpt-5.5", "gpt-5.4-mini", "gpt-5.3-codex"],
openai: ["gpt-5.5", "gpt-5.4-mini", "gpt-5.4-mini-fast", "gpt-5.3-codex"],
},
connected: ["anthropic", "google", "openai"],
updatedAt: "2026-01-01T00:00:00.000Z",
@@ -109,18 +110,18 @@ describe("sisyphus-task", () => {
// when / #then
expect(category).toBeDefined()
expect(category.model).toBe("openai/gpt-5.5")
expect(category.variant).toBe("medium")
expect(category.model).toBe("openai/gpt-5.3-codex")
expect(category.variant).toBe("high")
})
test("unspecified-high category uses claude-opus-4-7 max as primary", () => {
test("unspecified-high category uses gpt-5.5 high as primary", () => {
// given
const category = DEFAULT_CATEGORIES["unspecified-high"]
// when / #then
expect(category).toBeDefined()
expect(category.model).toBe("anthropic/claude-opus-4-7")
expect(category.variant).toBe("max")
expect(category.model).toBe("openai/gpt-5.5")
expect(category.variant).toBe("high")
})
})
@@ -680,7 +681,7 @@ describe("sisyphus-task", () => {
//#given - manager.launch returns before sessionID is available
const { createDelegateTask } = require("./tools")
const tasks = new Map<string, { id: string; sessionID?: string; status: string; description: string; agent: string }>()
const tasks = new Map<string, { id: string; sessionId?: string; status: string; description: string; agent: string }>()
const mockManager = {
getTask: (id: string) => tasks.get(id),
launch: async () => {
@@ -1081,7 +1082,7 @@ describe("sisyphus-task", () => {
abort: new AbortController().signal,
}
// when - unspecified-high uses claude-opus-4-7 max in DEFAULT_CATEGORIES
// when - unspecified-high uses gpt-5.5 high in DEFAULT_CATEGORIES
await tool.execute(
{
description: "Test unspecified-high default variant",
@@ -1093,11 +1094,11 @@ describe("sisyphus-task", () => {
toolContext
)
// then - claude-opus-4-7 should be passed with max variant
// then - gpt-5.5 should be passed with high variant
expect(launchInput.model).toEqual({
providerID: "anthropic",
modelID: "claude-opus-4-7",
variant: "max",
providerID: "openai",
modelID: "gpt-5.5",
variant: "high",
})
}, { timeout: 20000 })
@@ -1142,7 +1143,7 @@ describe("sisyphus-task", () => {
abort: new AbortController().signal,
}
// when - unspecified-high uses claude-opus-4-7 max in DEFAULT_CATEGORIES
// when - unspecified-high uses gpt-5.5 high in DEFAULT_CATEGORIES
await tool.execute(
{
description: "Test unspecified-high sync variant",
@@ -1154,12 +1155,12 @@ describe("sisyphus-task", () => {
toolContext
)
// then - claude-opus-4-7 should be passed with max variant
// then - gpt-5.5 should be passed with high variant
expect(promptBody.model).toEqual({
providerID: "anthropic",
modelID: "claude-opus-4-7",
providerID: "openai",
modelID: "gpt-5.5",
})
expect(promptBody.variant).toBe("max")
expect(promptBody.variant).toBe("high")
}, { timeout: 20000 })
})
@@ -2761,10 +2762,11 @@ describe("sisyphus-task", () => {
toolContext
)
// then - model should be openai/gpt-5.4-mini from DEFAULT_CATEGORIES
// then - model should be openai/gpt-5.4-mini-fast from DEFAULT_CATEGORIES
// NOT anthropic/claude-sonnet-4-6 (system default)
expect(launchInput.model.providerID).toBe("openai")
expect(launchInput.model.modelID).toBe("gpt-5.4-mini")
expect(launchInput.model.modelID).toBe("gpt-5.4-mini-fast")
expect(launchInput.model.variant).toBe("none")
})
test("category delegation ignores UI-selected (Kimi) system default model", async () => {
@@ -2828,7 +2830,8 @@ describe("sisyphus-task", () => {
// then - category model must win (not Kimi)
expect(launchInput.model.providerID).toBe("openai")
expect(launchInput.model.modelID).toBe("gpt-5.4-mini")
expect(launchInput.model.modelID).toBe("gpt-5.4-mini-fast")
expect(launchInput.model.variant).toBe("none")
})
test("sisyphus-junior model override takes precedence over category model", async () => {
@@ -3450,6 +3453,25 @@ describe("sisyphus-task", () => {
expect(resolved!.config.variant).toBe("xhigh")
})
test("requested built-in category model defaults stay aligned", () => {
// given - categories with explicit runtime defaults
const expectedDefaults = {
deep: { model: "openai/gpt-5.3-codex", variant: "high" },
quick: { model: "openai/gpt-5.4-mini-fast", variant: "none" },
"unspecified-low": { model: "openai/gpt-5.5", variant: "medium" },
"unspecified-high": { model: "openai/gpt-5.5", variant: "high" },
} satisfies Record<string, { model: string; variant: string }>
for (const [categoryName, expected] of Object.entries(expectedDefaults)) {
// when
const resolved = resolveCategoryConfig(categoryName, { systemDefaultModel: SYSTEM_DEFAULT_MODEL })
// then
expect(resolved?.config.model).toBe(expected.model)
expect(resolved?.config.variant).toBe(expected.variant)
}
})
test("default model is used for category with default entry", () => {
// given - unspecified-low has default model
const categoryName = "unspecified-low"
@@ -3459,7 +3481,8 @@ describe("sisyphus-task", () => {
// then - default model from DEFAULT_CATEGORIES is used
expect(resolved).not.toBeNull()
expect(resolved!.config.model).toBe("anthropic/claude-sonnet-4-6")
expect(resolved!.config.model).toBe("openai/gpt-5.5")
expect(resolved!.config.variant).toBe("medium")
})
test("category built-in model takes precedence over inheritedModel for builtin category", () => {