test(delegate-task): add failing tests for model variant preservation
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
@@ -4,6 +4,7 @@ import type { DelegateTaskArgs, ToolContextWithMetadata } from "./types"
|
|||||||
import type { ParentContext } from "./executor-types"
|
import type { ParentContext } from "./executor-types"
|
||||||
|
|
||||||
const MODEL = { providerID: "anthropic", modelID: "claude-sonnet-4-6" }
|
const MODEL = { providerID: "anthropic", modelID: "claude-sonnet-4-6" }
|
||||||
|
const MODEL_WITH_VARIANT = { providerID: "google", modelID: "gemini-3.1-pro", variant: "high" }
|
||||||
|
|
||||||
function makeMockCtx(): ToolContextWithMetadata & { captured: any[] } {
|
function makeMockCtx(): ToolContextWithMetadata & { captured: any[] } {
|
||||||
const captured: any[] = []
|
const captured: any[] = []
|
||||||
@@ -344,4 +345,148 @@ describe("metadata model unification", () => {
|
|||||||
expect(meta.metadata.model).toBeUndefined()
|
expect(meta.metadata.model).toBeUndefined()
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe("#given category model with variant", () => {
|
||||||
|
describe("#when executors publish metadata", () => {
|
||||||
|
test("#then sync-task metadata includes variant", async () => {
|
||||||
|
const { executeSyncTask } = require("./sync-task")
|
||||||
|
const ctx = makeMockCtx()
|
||||||
|
const deps = {
|
||||||
|
createSyncSession: async () => ({ ok: true, sessionID: "ses_sync_variant" }),
|
||||||
|
sendSyncPrompt: async () => null,
|
||||||
|
pollSyncSession: async () => null,
|
||||||
|
fetchSyncResult: async () => ({ ok: true as const, textContent: "done" }),
|
||||||
|
}
|
||||||
|
const args: DelegateTaskArgs = {
|
||||||
|
description: "test", prompt: "do it",
|
||||||
|
category: "visual-engineering", load_skills: [], run_in_background: false,
|
||||||
|
}
|
||||||
|
|
||||||
|
await executeSyncTask(args, ctx, {
|
||||||
|
client: { session: { create: async () => ({ data: { id: "ses_sync_variant" } }) } },
|
||||||
|
directory: "/tmp",
|
||||||
|
onSyncSessionCreated: null,
|
||||||
|
}, parentContext, "explore", MODEL_WITH_VARIANT, undefined, undefined, undefined, deps)
|
||||||
|
|
||||||
|
const meta = ctx.captured.find((metadataEvent: any) => metadataEvent.metadata?.sessionId)
|
||||||
|
expect(meta).toBeDefined()
|
||||||
|
expect(meta.metadata.model).toEqual(MODEL_WITH_VARIANT)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("#then background-task metadata includes variant", async () => {
|
||||||
|
const { executeBackgroundTask } = require("./background-task")
|
||||||
|
const ctx = makeMockCtx()
|
||||||
|
const args: DelegateTaskArgs = {
|
||||||
|
description: "test", prompt: "do it",
|
||||||
|
category: "visual-engineering", load_skills: [], run_in_background: true, subagent_type: "explore",
|
||||||
|
}
|
||||||
|
|
||||||
|
await executeBackgroundTask(args, ctx, {
|
||||||
|
manager: {
|
||||||
|
launch: async () => ({
|
||||||
|
id: "bg_variant", description: "test", agent: "explore",
|
||||||
|
status: "pending", sessionID: "ses_bg_variant", model: MODEL_WITH_VARIANT,
|
||||||
|
}),
|
||||||
|
getTask: () => undefined,
|
||||||
|
},
|
||||||
|
} as any, parentContext, "explore", MODEL_WITH_VARIANT, undefined)
|
||||||
|
|
||||||
|
const meta = ctx.captured.find((metadataEvent: any) => metadataEvent.metadata?.sessionId)
|
||||||
|
expect(meta).toBeDefined()
|
||||||
|
expect(meta.metadata.model).toEqual(MODEL_WITH_VARIANT)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("#then unstable-agent-task metadata includes variant", async () => {
|
||||||
|
const { executeUnstableAgentTask } = require("./unstable-agent-task")
|
||||||
|
const ctx = makeMockCtx()
|
||||||
|
const args: DelegateTaskArgs = {
|
||||||
|
description: "test", prompt: "do it",
|
||||||
|
category: "visual-engineering", load_skills: [], run_in_background: false,
|
||||||
|
}
|
||||||
|
|
||||||
|
const launchedTask = {
|
||||||
|
id: "bg_unstable_variant", description: "test", agent: "explore",
|
||||||
|
status: "completed", sessionID: "ses_unstable_variant", model: MODEL_WITH_VARIANT,
|
||||||
|
}
|
||||||
|
|
||||||
|
await executeUnstableAgentTask(
|
||||||
|
args, ctx,
|
||||||
|
{
|
||||||
|
manager: {
|
||||||
|
launch: async () => launchedTask,
|
||||||
|
getTask: () => launchedTask,
|
||||||
|
},
|
||||||
|
client: {
|
||||||
|
session: {
|
||||||
|
status: async () => ({ data: { ses_unstable_variant: { type: "idle" } } }),
|
||||||
|
messages: async () => ({
|
||||||
|
data: [{
|
||||||
|
info: { role: "assistant", time: { created: 1 } },
|
||||||
|
parts: [{ type: "text", text: "done" }],
|
||||||
|
}],
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
syncPollTimeoutMs: 100,
|
||||||
|
} as any,
|
||||||
|
parentContext, "explore", MODEL_WITH_VARIANT, undefined, "google/gemini-3.1-pro high",
|
||||||
|
)
|
||||||
|
|
||||||
|
const meta = ctx.captured.find((metadataEvent: any) => metadataEvent.metadata?.sessionId)
|
||||||
|
expect(meta).toBeDefined()
|
||||||
|
expect(meta.metadata.model).toEqual(MODEL_WITH_VARIANT)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("#then background-continuation metadata includes variant from task", async () => {
|
||||||
|
const { executeBackgroundContinuation } = require("./background-continuation")
|
||||||
|
const ctx = makeMockCtx()
|
||||||
|
const args: DelegateTaskArgs = {
|
||||||
|
description: "continue", prompt: "keep going",
|
||||||
|
load_skills: [], run_in_background: true, task_id: "ses_resumed_variant",
|
||||||
|
}
|
||||||
|
|
||||||
|
await executeBackgroundContinuation(args, ctx, {
|
||||||
|
manager: {
|
||||||
|
resume: async () => ({
|
||||||
|
id: "bg_resume_variant", description: "continue", agent: "explore",
|
||||||
|
status: "running", sessionID: "ses_resumed_variant", model: MODEL_WITH_VARIANT,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
} as any, parentContext)
|
||||||
|
|
||||||
|
const meta = ctx.captured.find((metadataEvent: any) => metadataEvent.metadata?.sessionId)
|
||||||
|
expect(meta).toBeDefined()
|
||||||
|
expect(meta.metadata.model).toEqual(MODEL_WITH_VARIANT)
|
||||||
|
})
|
||||||
|
|
||||||
|
test("#then sync-continuation metadata includes variant from resumed session", async () => {
|
||||||
|
const { executeSyncContinuation } = require("./sync-continuation")
|
||||||
|
const ctx = makeMockCtx()
|
||||||
|
const args: DelegateTaskArgs = {
|
||||||
|
description: "continue", prompt: "keep going",
|
||||||
|
load_skills: [], run_in_background: false, task_id: "ses_cont_variant",
|
||||||
|
}
|
||||||
|
|
||||||
|
const deps = {
|
||||||
|
pollSyncSession: async () => null,
|
||||||
|
fetchSyncResult: async () => ({ ok: true as const, textContent: "done" }),
|
||||||
|
}
|
||||||
|
|
||||||
|
await executeSyncContinuation(args, ctx, {
|
||||||
|
client: {
|
||||||
|
session: {
|
||||||
|
messages: async () => ({
|
||||||
|
data: [{ info: { agent: "explore", model: MODEL_WITH_VARIANT, providerID: "google", modelID: "gemini-3.1-pro" } }],
|
||||||
|
}),
|
||||||
|
prompt: async () => ({}),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as any, parentContext, deps)
|
||||||
|
|
||||||
|
const meta = ctx.captured.find((metadataEvent: any) => metadataEvent.metadata?.sessionId)
|
||||||
|
expect(meta).toBeDefined()
|
||||||
|
expect(meta.metadata.model).toEqual(MODEL_WITH_VARIANT)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -39,12 +39,55 @@ describe("resolveMetadataModel", () => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
describe("#given primary has extra fields", () => {
|
describe("#given primary has extra fields", () => {
|
||||||
test("#when resolving #then strips to providerID and modelID only", () => {
|
test("#when resolving #then preserves variant and strips unrelated fields", () => {
|
||||||
const extended = { providerID: "openai", modelID: "gpt-5.4", variant: "high", temperature: 0.7 } as const
|
const extended = { providerID: "openai", modelID: "gpt-5.4", variant: "high", temperature: 0.7 } as const
|
||||||
|
|
||||||
const result = resolveMetadataModel(extended, undefined)
|
const result = resolveMetadataModel(extended, undefined)
|
||||||
|
|
||||||
|
expect(result).toEqual({ providerID: "openai", modelID: "gpt-5.4", variant: "high" })
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe("#given primary has variant", () => {
|
||||||
|
test("#when resolving metadata model #then variant is preserved", () => {
|
||||||
|
const primary = { providerID: "google", modelID: "gemini-3.1-pro", variant: "high" }
|
||||||
|
|
||||||
|
const result = resolveMetadataModel(primary, undefined)
|
||||||
|
|
||||||
|
expect(result).toEqual({ providerID: "google", modelID: "gemini-3.1-pro", variant: "high" })
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe("#given primary lacks variant but fallback has variant", () => {
|
||||||
|
test("#when primary provided #then fallback variant is not used", () => {
|
||||||
|
const primary = { providerID: "google", modelID: "gemini-3.1-pro" }
|
||||||
|
const fallback = { providerID: "anthropic", modelID: "claude", variant: "max" }
|
||||||
|
|
||||||
|
const result = resolveMetadataModel(primary, fallback)
|
||||||
|
|
||||||
|
expect(result).toEqual({ providerID: "google", modelID: "gemini-3.1-pro" })
|
||||||
|
expect(result?.variant).toBeUndefined()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe("#given primary is undefined and fallback has variant", () => {
|
||||||
|
test("#when resolving metadata model #then fallback variant is preserved", () => {
|
||||||
|
const fallback = { providerID: "anthropic", modelID: "claude", variant: "max" }
|
||||||
|
|
||||||
|
const result = resolveMetadataModel(undefined, fallback)
|
||||||
|
|
||||||
|
expect(result).toEqual({ providerID: "anthropic", modelID: "claude", variant: "max" })
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe("#given both lack variant", () => {
|
||||||
|
test("#when resolving metadata model #then variant is not on result", () => {
|
||||||
|
const primary = { providerID: "openai", modelID: "gpt-5.4" }
|
||||||
|
|
||||||
|
const result = resolveMetadataModel(primary, undefined)
|
||||||
|
|
||||||
expect(result).toEqual({ providerID: "openai", modelID: "gpt-5.4" })
|
expect(result).toEqual({ providerID: "openai", modelID: "gpt-5.4" })
|
||||||
|
expect(result?.variant).toBeUndefined()
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
Reference in New Issue
Block a user