|
|
|
@@ -33,7 +33,8 @@ const TEST_AVAILABLE_MODELS = new Set([
|
|
|
|
|
"anthropic/claude-haiku-4-5",
|
|
|
|
|
"google/gemini-3.1-pro",
|
|
|
|
|
"google/gemini-3-flash",
|
|
|
|
|
"openai/gpt-5.4",
|
|
|
|
|
"openai/gpt-5.4-mini",
|
|
|
|
|
"openai/gpt-5.5",
|
|
|
|
|
"openai/gpt-5.3-codex",
|
|
|
|
|
])
|
|
|
|
|
|
|
|
|
@@ -68,7 +69,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
models: {
|
|
|
|
|
anthropic: ["claude-opus-4-7", "claude-sonnet-4-6", "claude-haiku-4-5"],
|
|
|
|
|
google: ["gemini-3.1-pro", "gemini-3-flash"],
|
|
|
|
|
openai: ["gpt-5.4", "gpt-5.3-codex"],
|
|
|
|
|
openai: ["gpt-5.5", "gpt-5.4-mini", "gpt-5.3-codex"],
|
|
|
|
|
},
|
|
|
|
|
connected: ["anthropic", "google", "openai"],
|
|
|
|
|
updatedAt: "2026-01-01T00:00:00.000Z",
|
|
|
|
@@ -98,7 +99,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
|
|
|
|
|
// when / #then
|
|
|
|
|
expect(category).toBeDefined()
|
|
|
|
|
expect(category.model).toBe("openai/gpt-5.4")
|
|
|
|
|
expect(category.model).toBe("openai/gpt-5.5")
|
|
|
|
|
expect(category.variant).toBe("xhigh")
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
@@ -108,7 +109,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
|
|
|
|
|
// when / #then
|
|
|
|
|
expect(category).toBeDefined()
|
|
|
|
|
expect(category.model).toBe("openai/gpt-5.4")
|
|
|
|
|
expect(category.model).toBe("openai/gpt-5.5")
|
|
|
|
|
expect(category.variant).toBe("medium")
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
@@ -380,7 +381,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
//#when
|
|
|
|
|
await tool.execute(args as unknown as DelegateTaskArgs, toolContext)
|
|
|
|
|
await tool.execute(args as DelegateTaskArgs, toolContext)
|
|
|
|
|
|
|
|
|
|
//#then
|
|
|
|
|
expect(args.load_skills).toEqual(["playwright", "git-master"])
|
|
|
|
@@ -443,7 +444,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
//#when
|
|
|
|
|
await tool.execute(args as unknown as DelegateTaskArgs, toolContext)
|
|
|
|
|
await tool.execute(args as DelegateTaskArgs, toolContext)
|
|
|
|
|
|
|
|
|
|
//#then
|
|
|
|
|
expect(args.load_skills).toEqual([])
|
|
|
|
@@ -876,7 +877,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
const categoryName = "my-custom"
|
|
|
|
|
const userCategories = {
|
|
|
|
|
"my-custom": {
|
|
|
|
|
model: "openai/gpt-5.4",
|
|
|
|
|
model: "openai/gpt-5.5",
|
|
|
|
|
temperature: 0.5,
|
|
|
|
|
prompt_append: "You are a custom agent",
|
|
|
|
|
},
|
|
|
|
@@ -887,7 +888,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
|
|
|
|
|
// then
|
|
|
|
|
expect(result).not.toBeNull()
|
|
|
|
|
expect(result!.config.model).toBe("openai/gpt-5.4")
|
|
|
|
|
expect(result!.config.model).toBe("openai/gpt-5.5")
|
|
|
|
|
expect(result!.config.temperature).toBe(0.5)
|
|
|
|
|
expect(result!.promptAppend).toBe("You are a custom agent")
|
|
|
|
|
})
|
|
|
|
@@ -926,7 +927,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
test("systemDefaultModel is used as fallback when custom category has no model", () => {
|
|
|
|
|
// given - custom category with no model defined
|
|
|
|
|
const categoryName = "my-custom-no-model"
|
|
|
|
|
const userCategories = { "my-custom-no-model": { temperature: 0.5 } } as unknown as Record<string, CategoryConfig>
|
|
|
|
|
const userCategories: Record<string, CategoryConfig> = { "my-custom-no-model": { temperature: 0.5 } }
|
|
|
|
|
const inheritedModel = "cliproxy/claude-opus-4-7"
|
|
|
|
|
|
|
|
|
|
// when
|
|
|
|
@@ -1000,7 +1001,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
manager: mockManager,
|
|
|
|
|
client: mockClient,
|
|
|
|
|
userCategories: {
|
|
|
|
|
ultrabrain: { model: "openai/gpt-5.4", variant: "xhigh" },
|
|
|
|
|
ultrabrain: { model: "openai/gpt-5.5", variant: "xhigh" },
|
|
|
|
|
},
|
|
|
|
|
connectedProvidersOverride: TEST_CONNECTED_PROVIDERS,
|
|
|
|
|
availableModelsOverride: createTestAvailableModels(),
|
|
|
|
@@ -1028,7 +1029,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
// then
|
|
|
|
|
expect(launchInput.model).toEqual({
|
|
|
|
|
providerID: "openai",
|
|
|
|
|
modelID: "gpt-5.4",
|
|
|
|
|
modelID: "gpt-5.5",
|
|
|
|
|
variant: "xhigh",
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
@@ -2447,7 +2448,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Use ultrabrain which uses gpt-5.4 (non-gemini)
|
|
|
|
|
// Use ultrabrain which uses gpt-5.5 (non-gemini)
|
|
|
|
|
const tool = createDelegateTask({
|
|
|
|
|
manager: mockManager,
|
|
|
|
|
client: mockClient,
|
|
|
|
@@ -2553,7 +2554,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
models: {
|
|
|
|
|
anthropic: ["claude-opus-4-7", "claude-sonnet-4-6", "claude-haiku-4-5"],
|
|
|
|
|
google: ["gemini-3.1-pro", "gemini-3-flash"],
|
|
|
|
|
openai: ["gpt-5.4", "gpt-5.3-codex"],
|
|
|
|
|
openai: ["gpt-5.5", "gpt-5.5", "gpt-5.3-codex"],
|
|
|
|
|
"kimi-for-coding": ["k2p5"],
|
|
|
|
|
},
|
|
|
|
|
connected: ["anthropic", "google", "openai", "kimi-for-coding"],
|
|
|
|
@@ -2664,7 +2665,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
client: mockClient,
|
|
|
|
|
userCategories: {
|
|
|
|
|
"my-unstable-cat": {
|
|
|
|
|
model: "openai/gpt-5.4",
|
|
|
|
|
model: "openai/gpt-5.5",
|
|
|
|
|
is_unstable_agent: true,
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
@@ -2746,7 +2747,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
abort: new AbortController().signal,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// when - using "quick" category which should use "anthropic/claude-haiku-4-5"
|
|
|
|
|
// when - using "quick" category which should use the catalog model
|
|
|
|
|
await tool.execute(
|
|
|
|
|
{
|
|
|
|
|
description: "Test category fallback",
|
|
|
|
@@ -2758,10 +2759,10 @@ describe("sisyphus-task", () => {
|
|
|
|
|
toolContext
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// then - model should be anthropic/claude-haiku-4-5 from DEFAULT_CATEGORIES
|
|
|
|
|
// then - model should be openai/gpt-5.4-mini from DEFAULT_CATEGORIES
|
|
|
|
|
// NOT anthropic/claude-sonnet-4-6 (system default)
|
|
|
|
|
expect(launchInput.model.providerID).toBe("anthropic")
|
|
|
|
|
expect(launchInput.model.modelID).toBe("claude-haiku-4-5")
|
|
|
|
|
expect(launchInput.model.providerID).toBe("openai")
|
|
|
|
|
expect(launchInput.model.modelID).toBe("gpt-5.4-mini")
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
test("category delegation ignores UI-selected (Kimi) system default model", async () => {
|
|
|
|
@@ -2811,7 +2812,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
abort: new AbortController().signal,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// when - using "quick" category which should use "anthropic/claude-haiku-4-5"
|
|
|
|
|
// when - using "quick" category which should use the catalog model
|
|
|
|
|
await tool.execute(
|
|
|
|
|
{
|
|
|
|
|
description: "UI model inheritance test",
|
|
|
|
@@ -2824,8 +2825,8 @@ describe("sisyphus-task", () => {
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// then - category model must win (not Kimi)
|
|
|
|
|
expect(launchInput.model.providerID).toBe("anthropic")
|
|
|
|
|
expect(launchInput.model.modelID).toBe("claude-haiku-4-5")
|
|
|
|
|
expect(launchInput.model.providerID).toBe("openai")
|
|
|
|
|
expect(launchInput.model.modelID).toBe("gpt-5.4-mini")
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
test("sisyphus-junior model override takes precedence over category model", async () => {
|
|
|
|
@@ -2872,7 +2873,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
abort: new AbortController().signal,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// when - using ultrabrain category (default model is openai/gpt-5.4)
|
|
|
|
|
// when - using ultrabrain category (default model is openai/gpt-5.5)
|
|
|
|
|
await tool.execute(
|
|
|
|
|
{
|
|
|
|
|
description: "Override precedence test",
|
|
|
|
@@ -2924,7 +2925,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
client: mockClient,
|
|
|
|
|
sisyphusJuniorModel: "anthropic/claude-sonnet-4-6",
|
|
|
|
|
userCategories: {
|
|
|
|
|
ultrabrain: { model: "openai/gpt-5.4" },
|
|
|
|
|
ultrabrain: { model: "openai/gpt-5.5" },
|
|
|
|
|
},
|
|
|
|
|
connectedProvidersOverride: TEST_CONNECTED_PROVIDERS,
|
|
|
|
|
availableModelsOverride: createTestAvailableModels(),
|
|
|
|
@@ -2951,7 +2952,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
|
|
|
|
|
// then - explicit category model should win
|
|
|
|
|
expect(launchInput.model.providerID).toBe("openai")
|
|
|
|
|
expect(launchInput.model.modelID).toBe("gpt-5.4")
|
|
|
|
|
expect(launchInput.model.modelID).toBe("gpt-5.5")
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
test("sisyphus-junior model override works with quick category (#1295)", async () => {
|
|
|
|
@@ -3048,7 +3049,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
const tool = createDelegateTask({
|
|
|
|
|
manager: mockManager,
|
|
|
|
|
client: mockClient,
|
|
|
|
|
sisyphusJuniorModel: "openai/gpt-5.4",
|
|
|
|
|
sisyphusJuniorModel: "openai/gpt-5.5",
|
|
|
|
|
userCategories: {
|
|
|
|
|
"my-custom": { temperature: 0.5 },
|
|
|
|
|
},
|
|
|
|
@@ -3075,7 +3076,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
|
|
|
|
|
// then - sisyphus-junior override model should be used as fallback
|
|
|
|
|
expect(launchInput.model.providerID).toBe("openai")
|
|
|
|
|
expect(launchInput.model.modelID).toBe("gpt-5.4")
|
|
|
|
|
expect(launchInput.model.modelID).toBe("gpt-5.5")
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
@@ -3446,7 +3447,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
|
|
|
|
|
// then - catalog model is used
|
|
|
|
|
expect(resolved).not.toBeNull()
|
|
|
|
|
expect(resolved!.config.model).toBe("openai/gpt-5.4")
|
|
|
|
|
expect(resolved!.config.model).toBe("openai/gpt-5.5")
|
|
|
|
|
expect(resolved!.config.variant).toBe("xhigh")
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
@@ -3470,10 +3471,10 @@ describe("sisyphus-task", () => {
|
|
|
|
|
// when
|
|
|
|
|
const resolved = resolveCategoryConfig(categoryName, { inheritedModel, systemDefaultModel: SYSTEM_DEFAULT_MODEL })
|
|
|
|
|
|
|
|
|
|
// then - category's built-in model wins (ultrabrain uses gpt-5.4)
|
|
|
|
|
// then - category's built-in model wins (ultrabrain uses gpt-5.5)
|
|
|
|
|
expect(resolved).not.toBeNull()
|
|
|
|
|
const actualModel = resolved!.config.model
|
|
|
|
|
expect(actualModel).toBe("openai/gpt-5.4")
|
|
|
|
|
expect(actualModel).toBe("openai/gpt-5.5")
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
test("when user defines model - modelInfo should report user-defined regardless of inheritedModel", () => {
|
|
|
|
@@ -3527,18 +3528,18 @@ describe("sisyphus-task", () => {
|
|
|
|
|
const categoryName = "ultrabrain"
|
|
|
|
|
const inheritedModel = "anthropic/claude-opus-4-7"
|
|
|
|
|
|
|
|
|
|
// when category has a built-in model (gpt-5.4 for ultrabrain)
|
|
|
|
|
// when category has a built-in model (gpt-5.5 for ultrabrain)
|
|
|
|
|
const resolved = resolveCategoryConfig(categoryName, { inheritedModel, systemDefaultModel: SYSTEM_DEFAULT_MODEL })
|
|
|
|
|
|
|
|
|
|
// then category's built-in model should be used, NOT inheritedModel
|
|
|
|
|
expect(resolved).not.toBeNull()
|
|
|
|
|
expect(resolved!.model).toBe("openai/gpt-5.4")
|
|
|
|
|
expect(resolved!.model).toBe("openai/gpt-5.5")
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
test("FIXED: systemDefaultModel is used when no userConfig.model and no inheritedModel", () => {
|
|
|
|
|
// given a custom category with no default model
|
|
|
|
|
const categoryName = "custom-no-default"
|
|
|
|
|
const userCategories = { "custom-no-default": { temperature: 0.5 } } as unknown as Record<string, CategoryConfig>
|
|
|
|
|
const userCategories: Record<string, CategoryConfig> = { "custom-no-default": { temperature: 0.5 } }
|
|
|
|
|
const systemDefaultModel = "anthropic/claude-sonnet-4-6"
|
|
|
|
|
|
|
|
|
|
// when no inheritedModel is provided, only systemDefaultModel
|
|
|
|
@@ -3588,8 +3589,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
test("FIXED: undefined userConfig.model falls back to category built-in model", () => {
|
|
|
|
|
// given user sets a builtin category but leaves model undefined
|
|
|
|
|
const categoryName = "visual-engineering"
|
|
|
|
|
// Using type assertion since we're testing fallback behavior for categories without model
|
|
|
|
|
const userCategories = { "visual-engineering": { temperature: 0.2 } } as unknown as Record<string, CategoryConfig>
|
|
|
|
|
const userCategories: Record<string, CategoryConfig> = { "visual-engineering": { temperature: 0.2 } }
|
|
|
|
|
const inheritedModel = "anthropic/claude-opus-4-7"
|
|
|
|
|
|
|
|
|
|
// when resolveCategoryConfig is called
|
|
|
|
@@ -3603,8 +3603,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
test("systemDefaultModel is used when no other model is available", () => {
|
|
|
|
|
// given - custom category with no model, but systemDefaultModel is set
|
|
|
|
|
const categoryName = "my-custom"
|
|
|
|
|
// Using type assertion since we're testing fallback behavior for categories without model
|
|
|
|
|
const userCategories = { "my-custom": { temperature: 0.5 } } as unknown as Record<string, CategoryConfig>
|
|
|
|
|
const userCategories: Record<string, CategoryConfig> = { "my-custom": { temperature: 0.5 } }
|
|
|
|
|
const systemDefaultModel = "anthropic/claude-sonnet-4-6"
|
|
|
|
|
|
|
|
|
|
// when
|
|
|
|
@@ -3935,7 +3934,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
app: {
|
|
|
|
|
agents: async () => ({
|
|
|
|
|
data: [
|
|
|
|
|
{ name: "oracle", mode: "subagent", model: { providerID: "openai", modelID: "gpt-5.4" } },
|
|
|
|
|
{ name: "oracle", mode: "subagent", model: { providerID: "openai", modelID: "gpt-5.5" } },
|
|
|
|
|
],
|
|
|
|
|
}),
|
|
|
|
|
},
|
|
|
|
@@ -4002,7 +4001,7 @@ describe("sisyphus-task", () => {
|
|
|
|
|
app: {
|
|
|
|
|
agents: async () => ({
|
|
|
|
|
data: [
|
|
|
|
|
{ name: "oracle", mode: "subagent", model: { providerID: "openai", modelID: "gpt-5.4" } },
|
|
|
|
|
{ name: "oracle", mode: "subagent", model: { providerID: "openai", modelID: "gpt-5.5" } },
|
|
|
|
|
],
|
|
|
|
|
}),
|
|
|
|
|
},
|
|
|
|
@@ -4111,11 +4110,11 @@ describe("sisyphus-task", () => {
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// then - should resolve via AGENT_MODEL_REQUIREMENTS fallback chain for oracle
|
|
|
|
|
// oracle fallback chain: gpt-5.4 (openai) > gemini-3.1-pro (google) > claude-opus-4-7 (anthropic)
|
|
|
|
|
// Since openai is in connectedProviders, should resolve to openai/gpt-5.4
|
|
|
|
|
// oracle fallback chain: gpt-5.5 (openai) > gemini-3.1-pro (google) > claude-opus-4-7 (anthropic)
|
|
|
|
|
// Since openai is in connectedProviders, should resolve to openai/gpt-5.5
|
|
|
|
|
expect(promptBody.model).toBeDefined()
|
|
|
|
|
expect(promptBody.model.providerID).toBe("openai")
|
|
|
|
|
expect(promptBody.model.modelID).toContain("gpt-5.4")
|
|
|
|
|
expect(promptBody.model.modelID).toContain("gpt-5.5")
|
|
|
|
|
}, { timeout: 20000 })
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|