feat(model-settings-compat): add variant/reasoningEffort compatibility resolver
- Registry-based model family detection (provider-agnostic) - Variant and reasoningEffort ladder downgrade logic - Three-tier resolution: metadata override → family heuristic → unknown drop - Comprehensive test suite covering all model families
This commit is contained in:
Ravi Tharuma
@@ -43,6 +43,7 @@ export type {
|
||||
ModelResolutionResult,
|
||||
} from "./model-resolution-types"
|
||||
export * from "./model-availability"
|
||||
export * from "./model-settings-compatibility"
|
||||
export * from "./fallback-model-availability"
|
||||
export * from "./connected-providers-cache"
|
||||
export * from "./context-limit-resolver"
|
||||
|
||||
@@ -0,0 +1,302 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
|
||||
import { resolveCompatibleModelSettings } from "./model-settings-compatibility"
|
||||
|
||||
describe("resolveCompatibleModelSettings", () => {
|
||||
test("keeps supported Claude Opus variant unchanged", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "anthropic",
|
||||
modelID: "claude-opus-4-6",
|
||||
desired: { variant: "max" },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: "max",
|
||||
reasoningEffort: undefined,
|
||||
changes: [],
|
||||
})
|
||||
})
|
||||
|
||||
test("uses model metadata first for variant support", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "anthropic",
|
||||
modelID: "claude-opus-4-6",
|
||||
desired: { variant: "max" },
|
||||
capabilities: { variants: ["low", "medium", "high"] },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: "high",
|
||||
reasoningEffort: undefined,
|
||||
changes: [
|
||||
{
|
||||
field: "variant",
|
||||
from: "max",
|
||||
to: "high",
|
||||
reason: "unsupported-by-model-metadata",
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
test("prefers metadata over family heuristics even when family would allow a higher level", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "anthropic",
|
||||
modelID: "claude-opus-4-6",
|
||||
desired: { variant: "max" },
|
||||
capabilities: { variants: ["low", "medium"] },
|
||||
})
|
||||
|
||||
expect(result.variant).toBe("medium")
|
||||
expect(result.changes).toEqual([
|
||||
{
|
||||
field: "variant",
|
||||
from: "max",
|
||||
to: "medium",
|
||||
reason: "unsupported-by-model-metadata",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("downgrades unsupported Claude Sonnet max variant to high when metadata is absent", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "anthropic",
|
||||
modelID: "claude-sonnet-4-6",
|
||||
desired: { variant: "max" },
|
||||
})
|
||||
|
||||
expect(result.variant).toBe("high")
|
||||
expect(result.changes).toEqual([
|
||||
{
|
||||
field: "variant",
|
||||
from: "max",
|
||||
to: "high",
|
||||
reason: "unsupported-by-model-family",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("keeps supported GPT reasoningEffort unchanged", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "gpt-5.4",
|
||||
desired: { reasoningEffort: "high" },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: undefined,
|
||||
reasoningEffort: "high",
|
||||
changes: [],
|
||||
})
|
||||
})
|
||||
|
||||
test("downgrades gpt-5 reasoningEffort max to xhigh", () => {
|
||||
// given
|
||||
const input = {
|
||||
providerID: "openai",
|
||||
modelID: "gpt-5.4",
|
||||
desired: { reasoningEffort: "max" },
|
||||
}
|
||||
|
||||
// when
|
||||
const result = resolveCompatibleModelSettings(input)
|
||||
|
||||
// then
|
||||
expect(result.reasoningEffort).toBe("xhigh")
|
||||
expect(result.changes).toEqual([
|
||||
{
|
||||
field: "reasoningEffort",
|
||||
from: "max",
|
||||
to: "xhigh",
|
||||
reason: "unsupported-by-model-family",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("keeps supported OpenAI reasoning-family effort for o-series models", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "o3-mini",
|
||||
desired: { reasoningEffort: "high" },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: undefined,
|
||||
reasoningEffort: "high",
|
||||
changes: [],
|
||||
})
|
||||
})
|
||||
|
||||
test("downgrades openai reasoning-family effort xhigh to high", () => {
|
||||
// given
|
||||
const input = {
|
||||
providerID: "openai",
|
||||
modelID: "o3-mini",
|
||||
desired: { reasoningEffort: "xhigh" },
|
||||
}
|
||||
|
||||
// when
|
||||
const result = resolveCompatibleModelSettings(input)
|
||||
|
||||
// then
|
||||
expect(result.reasoningEffort).toBe("high")
|
||||
expect(result.changes).toEqual([
|
||||
{
|
||||
field: "reasoningEffort",
|
||||
from: "xhigh",
|
||||
to: "high",
|
||||
reason: "unsupported-by-model-family",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("drops reasoningEffort for gpt-5 mini models", () => {
|
||||
// given
|
||||
const input = {
|
||||
providerID: "openai",
|
||||
modelID: "gpt-5.4-mini",
|
||||
desired: { reasoningEffort: "high" },
|
||||
}
|
||||
|
||||
// when
|
||||
const result = resolveCompatibleModelSettings(input)
|
||||
|
||||
// then
|
||||
expect(result.reasoningEffort).toBeUndefined()
|
||||
expect(result.changes).toEqual([
|
||||
{
|
||||
field: "reasoningEffort",
|
||||
from: "high",
|
||||
to: undefined,
|
||||
reason: "unsupported-by-model-family",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("treats non-openai o-series models as unknown", () => {
|
||||
// given
|
||||
const input = {
|
||||
providerID: "ollama",
|
||||
modelID: "o3",
|
||||
desired: { reasoningEffort: "high" },
|
||||
}
|
||||
|
||||
// when
|
||||
const result = resolveCompatibleModelSettings(input)
|
||||
|
||||
// then
|
||||
expect(result.reasoningEffort).toBeUndefined()
|
||||
expect(result.changes).toEqual([
|
||||
{
|
||||
field: "reasoningEffort",
|
||||
from: "high",
|
||||
to: undefined,
|
||||
reason: "unknown-model-family",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("does not record case-only normalization as a compatibility downgrade", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "gpt-5.4",
|
||||
desired: { variant: "HIGH", reasoningEffort: "HIGH" },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: "high",
|
||||
reasoningEffort: "high",
|
||||
changes: [],
|
||||
})
|
||||
})
|
||||
|
||||
test("downgrades unsupported GPT reasoningEffort to nearest lower level", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "gpt-4.1",
|
||||
desired: { reasoningEffort: "xhigh" },
|
||||
})
|
||||
|
||||
expect(result.reasoningEffort).toBe("high")
|
||||
expect(result.changes).toEqual([
|
||||
{
|
||||
field: "reasoningEffort",
|
||||
from: "xhigh",
|
||||
to: "high",
|
||||
reason: "unsupported-by-model-family",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("drops reasoningEffort for Claude family", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "anthropic",
|
||||
modelID: "claude-sonnet-4-6",
|
||||
desired: { reasoningEffort: "high" },
|
||||
})
|
||||
|
||||
expect(result.reasoningEffort).toBeUndefined()
|
||||
expect(result.changes).toEqual([
|
||||
{
|
||||
field: "reasoningEffort",
|
||||
from: "high",
|
||||
to: undefined,
|
||||
reason: "unsupported-by-model-family",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("handles combined variant and reasoningEffort normalization", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "anthropic",
|
||||
modelID: "claude-sonnet-4-6",
|
||||
desired: { variant: "max", reasoningEffort: "high" },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: "high",
|
||||
reasoningEffort: undefined,
|
||||
changes: [
|
||||
{
|
||||
field: "variant",
|
||||
from: "max",
|
||||
to: "high",
|
||||
reason: "unsupported-by-model-family",
|
||||
},
|
||||
{
|
||||
field: "reasoningEffort",
|
||||
from: "high",
|
||||
to: undefined,
|
||||
reason: "unsupported-by-model-family",
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
test("treats unknown model families conservatively by dropping unsupported settings", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "mystery",
|
||||
modelID: "mystery-model-1",
|
||||
desired: { variant: "max", reasoningEffort: "high" },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: undefined,
|
||||
reasoningEffort: undefined,
|
||||
changes: [
|
||||
{
|
||||
field: "variant",
|
||||
from: "max",
|
||||
to: undefined,
|
||||
reason: "unknown-model-family",
|
||||
},
|
||||
{
|
||||
field: "reasoningEffort",
|
||||
from: "high",
|
||||
to: undefined,
|
||||
reason: "unknown-model-family",
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,260 @@
|
||||
import { normalizeModelID } from "./model-normalization"
|
||||
|
||||
type CompatibilityField = "variant" | "reasoningEffort"
|
||||
|
||||
type DesiredModelSettings = {
|
||||
variant?: string
|
||||
reasoningEffort?: string
|
||||
}
|
||||
|
||||
type VariantCapabilities = {
|
||||
variants?: string[]
|
||||
}
|
||||
|
||||
export type ModelSettingsCompatibilityInput = {
|
||||
providerID: string
|
||||
modelID: string
|
||||
desired: DesiredModelSettings
|
||||
capabilities?: VariantCapabilities
|
||||
}
|
||||
|
||||
export type ModelSettingsCompatibilityChange = {
|
||||
field: CompatibilityField
|
||||
from: string
|
||||
to?: string
|
||||
reason: "unsupported-by-model-family" | "unknown-model-family" | "unsupported-by-model-metadata"
|
||||
}
|
||||
|
||||
export type ModelSettingsCompatibilityResult = {
|
||||
variant?: string
|
||||
reasoningEffort?: string
|
||||
changes: ModelSettingsCompatibilityChange[]
|
||||
}
|
||||
|
||||
type ModelFamily = "claude-opus" | "claude-non-opus" | "openai-reasoning" | "gpt-5" | "gpt-5-mini" | "gpt-legacy" | "unknown"
|
||||
|
||||
const VARIANT_LADDER = ["low", "medium", "high", "xhigh", "max"]
|
||||
const REASONING_LADDER = ["none", "minimal", "low", "medium", "high", "xhigh", "max"]
|
||||
|
||||
function detectModelFamily(providerID: string, modelID: string): ModelFamily {
|
||||
const provider = providerID.toLowerCase()
|
||||
const model = normalizeModelID(modelID).toLowerCase()
|
||||
|
||||
const isClaudeProvider = [
|
||||
"anthropic",
|
||||
"google-vertex-anthropic",
|
||||
"aws-bedrock-anthropic",
|
||||
].includes(provider)
|
||||
|| (["github-copilot", "opencode", "aws-bedrock", "bedrock"].includes(provider) && model.includes("claude"))
|
||||
|
||||
if (isClaudeProvider) {
|
||||
return /claude(?:-\d+(?:-\d+)*)?-opus/.test(model) ? "claude-opus" : "claude-non-opus"
|
||||
}
|
||||
|
||||
const isOpenAiReasoningFamily = provider === "openai" && (/^o\d(?:$|-)/.test(model) || model.includes("reasoning"))
|
||||
if (isOpenAiReasoningFamily) {
|
||||
return "openai-reasoning"
|
||||
}
|
||||
|
||||
if (/gpt-5.*-mini/.test(model)) {
|
||||
return "gpt-5-mini"
|
||||
}
|
||||
|
||||
if (model.includes("gpt-5")) {
|
||||
return "gpt-5"
|
||||
}
|
||||
|
||||
if (model.includes("gpt") || (provider === "openai" && /^o\d(?:$|-)/.test(model))) {
|
||||
return "gpt-legacy"
|
||||
}
|
||||
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
function downgradeWithinLadder(value: string, allowed: string[], ladder: string[]): string | undefined {
|
||||
const requestedIndex = ladder.indexOf(value)
|
||||
if (requestedIndex === -1) return undefined
|
||||
|
||||
for (let index = requestedIndex; index >= 0; index -= 1) {
|
||||
const candidate = ladder[index]
|
||||
if (allowed.includes(candidate)) {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
function normalizeCapabilitiesVariants(capabilities: VariantCapabilities | undefined): string[] | undefined {
|
||||
if (!capabilities?.variants || capabilities.variants.length === 0) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return capabilities.variants.map((variant) => variant.toLowerCase())
|
||||
}
|
||||
|
||||
function resolveVariant(
|
||||
modelFamily: ModelFamily,
|
||||
variant: string,
|
||||
capabilities?: VariantCapabilities,
|
||||
): { value?: string; reason?: ModelSettingsCompatibilityChange["reason"] } {
|
||||
const normalized = variant.toLowerCase()
|
||||
const metadataVariants = normalizeCapabilitiesVariants(capabilities)
|
||||
|
||||
if (metadataVariants) {
|
||||
if (metadataVariants.includes(normalized)) {
|
||||
return { value: normalized }
|
||||
}
|
||||
return {
|
||||
value: downgradeWithinLadder(normalized, metadataVariants, VARIANT_LADDER),
|
||||
reason: "unsupported-by-model-metadata",
|
||||
}
|
||||
}
|
||||
|
||||
if (modelFamily === "claude-opus") {
|
||||
const allowed = ["low", "medium", "high", "max"]
|
||||
if (allowed.includes(normalized)) {
|
||||
return { value: normalized }
|
||||
}
|
||||
return {
|
||||
value: downgradeWithinLadder(normalized, allowed, VARIANT_LADDER),
|
||||
reason: "unsupported-by-model-family",
|
||||
}
|
||||
}
|
||||
|
||||
if (modelFamily === "claude-non-opus") {
|
||||
const allowed = ["low", "medium", "high"]
|
||||
if (allowed.includes(normalized)) {
|
||||
return { value: normalized }
|
||||
}
|
||||
return {
|
||||
value: downgradeWithinLadder(normalized, allowed, VARIANT_LADDER),
|
||||
reason: "unsupported-by-model-family",
|
||||
}
|
||||
}
|
||||
|
||||
if (modelFamily === "gpt-5" || modelFamily === "gpt-5-mini") {
|
||||
const allowed = ["low", "medium", "high", "xhigh", "max"]
|
||||
if (allowed.includes(normalized)) {
|
||||
return { value: normalized }
|
||||
}
|
||||
return {
|
||||
value: downgradeWithinLadder(normalized, allowed, VARIANT_LADDER),
|
||||
reason: "unsupported-by-model-family",
|
||||
}
|
||||
}
|
||||
|
||||
if (modelFamily === "openai-reasoning") {
|
||||
const allowed = ["low", "medium", "high"]
|
||||
if (allowed.includes(normalized)) {
|
||||
return { value: normalized }
|
||||
}
|
||||
return {
|
||||
value: downgradeWithinLadder(normalized, allowed, VARIANT_LADDER),
|
||||
reason: "unsupported-by-model-family",
|
||||
}
|
||||
}
|
||||
|
||||
if (modelFamily === "gpt-legacy") {
|
||||
const allowed = ["low", "medium", "high"]
|
||||
if (allowed.includes(normalized)) {
|
||||
return { value: normalized }
|
||||
}
|
||||
return {
|
||||
value: downgradeWithinLadder(normalized, allowed, VARIANT_LADDER),
|
||||
reason: "unsupported-by-model-family",
|
||||
}
|
||||
}
|
||||
|
||||
return { value: undefined, reason: "unknown-model-family" }
|
||||
}
|
||||
|
||||
function resolveReasoningEffort(modelFamily: ModelFamily, reasoningEffort: string): { value?: string; reason?: ModelSettingsCompatibilityChange["reason"] } {
|
||||
const normalized = reasoningEffort.toLowerCase()
|
||||
|
||||
if (modelFamily === "gpt-5") {
|
||||
const allowed = ["none", "minimal", "low", "medium", "high", "xhigh"]
|
||||
if (allowed.includes(normalized)) {
|
||||
return { value: normalized }
|
||||
}
|
||||
return {
|
||||
value: downgradeWithinLadder(normalized, allowed, REASONING_LADDER),
|
||||
reason: "unsupported-by-model-family",
|
||||
}
|
||||
}
|
||||
|
||||
if (modelFamily === "gpt-5-mini") {
|
||||
return { value: undefined, reason: "unsupported-by-model-family" }
|
||||
}
|
||||
|
||||
if (modelFamily === "openai-reasoning") {
|
||||
const allowed = ["none", "minimal", "low", "medium", "high"]
|
||||
if (allowed.includes(normalized)) {
|
||||
return { value: normalized }
|
||||
}
|
||||
return {
|
||||
value: downgradeWithinLadder(normalized, allowed, REASONING_LADDER),
|
||||
reason: "unsupported-by-model-family",
|
||||
}
|
||||
}
|
||||
|
||||
if (modelFamily === "gpt-legacy") {
|
||||
const allowed = ["none", "minimal", "low", "medium", "high"]
|
||||
if (allowed.includes(normalized)) {
|
||||
return { value: normalized }
|
||||
}
|
||||
return {
|
||||
value: downgradeWithinLadder(normalized, allowed, REASONING_LADDER),
|
||||
reason: "unsupported-by-model-family",
|
||||
}
|
||||
}
|
||||
|
||||
if (modelFamily === "claude-opus" || modelFamily === "claude-non-opus") {
|
||||
return { value: undefined, reason: "unsupported-by-model-family" }
|
||||
}
|
||||
|
||||
return { value: undefined, reason: "unknown-model-family" }
|
||||
}
|
||||
|
||||
export function resolveCompatibleModelSettings(
|
||||
input: ModelSettingsCompatibilityInput,
|
||||
): ModelSettingsCompatibilityResult {
|
||||
const modelFamily = detectModelFamily(input.providerID, input.modelID)
|
||||
const changes: ModelSettingsCompatibilityChange[] = []
|
||||
|
||||
let variant = input.desired.variant
|
||||
if (variant !== undefined) {
|
||||
const normalizedVariant = variant.toLowerCase()
|
||||
const resolved = resolveVariant(modelFamily, normalizedVariant, input.capabilities)
|
||||
if (resolved.value !== normalizedVariant && resolved.reason) {
|
||||
changes.push({
|
||||
field: "variant",
|
||||
from: variant,
|
||||
to: resolved.value,
|
||||
reason: resolved.reason,
|
||||
})
|
||||
}
|
||||
variant = resolved.value
|
||||
}
|
||||
|
||||
let reasoningEffort = input.desired.reasoningEffort
|
||||
if (reasoningEffort !== undefined) {
|
||||
const normalizedReasoningEffort = reasoningEffort.toLowerCase()
|
||||
const resolved = resolveReasoningEffort(modelFamily, normalizedReasoningEffort)
|
||||
if (resolved.value !== normalizedReasoningEffort && resolved.reason) {
|
||||
changes.push({
|
||||
field: "reasoningEffort",
|
||||
from: reasoningEffort,
|
||||
to: resolved.value,
|
||||
reason: resolved.reason,
|
||||
})
|
||||
}
|
||||
reasoningEffort = resolved.value
|
||||
}
|
||||
|
||||
return {
|
||||
variant,
|
||||
reasoningEffort,
|
||||
changes,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import { clearSessionPromptParams, setSessionPromptParams } from "./session-prompt-params-state"
|
||||
|
||||
type PromptParamModel = {
|
||||
temperature?: number
|
||||
top_p?: number
|
||||
reasoningEffort?: string
|
||||
maxTokens?: number
|
||||
thinking?: { type: "enabled" | "disabled"; budgetTokens?: number }
|
||||
}
|
||||
|
||||
export function applySessionPromptParams(
|
||||
sessionID: string,
|
||||
model: PromptParamModel | undefined,
|
||||
): void {
|
||||
if (!model) {
|
||||
clearSessionPromptParams(sessionID)
|
||||
return
|
||||
}
|
||||
|
||||
const promptOptions: Record<string, unknown> = {
|
||||
...(model.reasoningEffort ? { reasoningEffort: model.reasoningEffort } : {}),
|
||||
...(model.thinking ? { thinking: model.thinking } : {}),
|
||||
...(model.maxTokens !== undefined ? { maxTokens: model.maxTokens } : {}),
|
||||
}
|
||||
|
||||
setSessionPromptParams(sessionID, {
|
||||
...(model.temperature !== undefined ? { temperature: model.temperature } : {}),
|
||||
...(model.top_p !== undefined ? { topP: model.top_p } : {}),
|
||||
...(Object.keys(promptOptions).length > 0 ? { options: promptOptions } : {}),
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user