Merge origin/dev into feature/upgrade-minimax-m2.7 (resolve conflicts)
This commit is contained in:
@@ -7,6 +7,7 @@ import { tmpdir } from "node:os"
|
||||
import { join } from "node:path"
|
||||
import {
|
||||
createConnectedProvidersCacheStore,
|
||||
findProviderModelMetadata,
|
||||
} from "./connected-providers-cache"
|
||||
|
||||
let fakeUserCacheRoot = ""
|
||||
@@ -68,8 +69,14 @@ describe("updateConnectedProvidersCache", () => {
|
||||
expect(cache).not.toBeNull()
|
||||
expect(cache!.connected).toEqual(["openai", "anthropic"])
|
||||
expect(cache!.models).toEqual({
|
||||
openai: ["gpt-5.3-codex", "gpt-5.4"],
|
||||
anthropic: ["claude-opus-4-6", "claude-sonnet-4-6"],
|
||||
openai: [
|
||||
{ id: "gpt-5.3-codex", name: "GPT-5.3 Codex" },
|
||||
{ id: "gpt-5.4", name: "GPT-5.4" },
|
||||
],
|
||||
anthropic: [
|
||||
{ id: "claude-opus-4-6", name: "Claude Opus 4.6" },
|
||||
{ id: "claude-sonnet-4-6", name: "Claude Sonnet 4.6" },
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
@@ -174,4 +181,86 @@ describe("updateConnectedProvidersCache", () => {
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test("findProviderModelMetadata returns rich cached metadata", async () => {
|
||||
//#given
|
||||
const mockClient = {
|
||||
provider: {
|
||||
list: async () => ({
|
||||
data: {
|
||||
connected: ["openai"],
|
||||
all: [
|
||||
{
|
||||
id: "openai",
|
||||
models: {
|
||||
"gpt-5.4": {
|
||||
id: "gpt-5.4",
|
||||
name: "GPT-5.4",
|
||||
temperature: false,
|
||||
variants: {
|
||||
low: {},
|
||||
high: {},
|
||||
},
|
||||
limit: { output: 128000 },
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
await testCacheStore.updateConnectedProvidersCache(mockClient)
|
||||
const cache = testCacheStore.readProviderModelsCache()
|
||||
|
||||
//#when
|
||||
const result = findProviderModelMetadata("openai", "gpt-5.4", cache)
|
||||
|
||||
//#then
|
||||
expect(result).toEqual({
|
||||
id: "gpt-5.4",
|
||||
name: "GPT-5.4",
|
||||
temperature: false,
|
||||
variants: {
|
||||
low: {},
|
||||
high: {},
|
||||
},
|
||||
limit: { output: 128000 },
|
||||
})
|
||||
})
|
||||
|
||||
test("keeps normalized fallback ids when raw metadata id is not a string", async () => {
|
||||
const mockClient = {
|
||||
provider: {
|
||||
list: async () => ({
|
||||
data: {
|
||||
connected: ["openai"],
|
||||
all: [
|
||||
{
|
||||
id: "openai",
|
||||
models: {
|
||||
"o3-mini": {
|
||||
id: 123,
|
||||
name: "o3-mini",
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
await testCacheStore.updateConnectedProvidersCache(mockClient)
|
||||
const cache = testCacheStore.readProviderModelsCache()
|
||||
|
||||
expect(cache?.models.openai).toEqual([
|
||||
{ id: "o3-mini", name: "o3-mini" },
|
||||
])
|
||||
expect(findProviderModelMetadata("openai", "o3-mini", cache)).toEqual({
|
||||
id: "o3-mini",
|
||||
name: "o3-mini",
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -11,20 +11,39 @@ interface ConnectedProvidersCache {
|
||||
updatedAt: string
|
||||
}
|
||||
|
||||
interface ModelMetadata {
|
||||
export interface ModelMetadata {
|
||||
id: string
|
||||
provider?: string
|
||||
context?: number
|
||||
output?: number
|
||||
name?: string
|
||||
variants?: Record<string, unknown>
|
||||
limit?: {
|
||||
context?: number
|
||||
input?: number
|
||||
output?: number
|
||||
}
|
||||
modalities?: {
|
||||
input?: string[]
|
||||
output?: string[]
|
||||
}
|
||||
capabilities?: Record<string, unknown>
|
||||
reasoning?: boolean
|
||||
temperature?: boolean
|
||||
tool_call?: boolean
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
interface ProviderModelsCache {
|
||||
export interface ProviderModelsCache {
|
||||
models: Record<string, string[] | ModelMetadata[]>
|
||||
connected: string[]
|
||||
updatedAt: string
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null
|
||||
}
|
||||
|
||||
export function createConnectedProvidersCacheStore(
|
||||
getCacheDir: () => string = dataPath.getOmoOpenCodeCacheDir
|
||||
) {
|
||||
@@ -119,7 +138,7 @@ export function createConnectedProvidersCacheStore(
|
||||
return existsSync(cacheFile)
|
||||
}
|
||||
|
||||
function writeProviderModelsCache(data: { models: Record<string, string[]>; connected: string[] }): void {
|
||||
function writeProviderModelsCache(data: { models: Record<string, string[] | ModelMetadata[]>; connected: string[] }): void {
|
||||
ensureCacheDir()
|
||||
const cacheFile = getCacheFilePath(PROVIDER_MODELS_CACHE_FILE)
|
||||
|
||||
@@ -164,14 +183,27 @@ export function createConnectedProvidersCacheStore(
|
||||
|
||||
writeConnectedProvidersCache(connected)
|
||||
|
||||
const modelsByProvider: Record<string, string[]> = {}
|
||||
const modelsByProvider: Record<string, ModelMetadata[]> = {}
|
||||
const allProviders = result.data?.all ?? []
|
||||
|
||||
for (const provider of allProviders) {
|
||||
if (provider.models) {
|
||||
const modelIds = Object.keys(provider.models)
|
||||
if (modelIds.length > 0) {
|
||||
modelsByProvider[provider.id] = modelIds
|
||||
const modelMetadata = Object.entries(provider.models).map(([modelID, rawMetadata]) => {
|
||||
if (!isRecord(rawMetadata)) {
|
||||
return { id: modelID }
|
||||
}
|
||||
|
||||
const normalizedID = typeof rawMetadata.id === "string"
|
||||
? rawMetadata.id
|
||||
: modelID
|
||||
|
||||
return {
|
||||
...rawMetadata,
|
||||
id: normalizedID,
|
||||
} satisfies ModelMetadata
|
||||
})
|
||||
if (modelMetadata.length > 0) {
|
||||
modelsByProvider[provider.id] = modelMetadata
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -200,6 +232,32 @@ export function createConnectedProvidersCacheStore(
|
||||
}
|
||||
}
|
||||
|
||||
export function findProviderModelMetadata(
|
||||
providerID: string,
|
||||
modelID: string,
|
||||
cache: ProviderModelsCache | null = defaultConnectedProvidersCacheStore.readProviderModelsCache(),
|
||||
): ModelMetadata | undefined {
|
||||
const providerModels = cache?.models?.[providerID]
|
||||
if (!providerModels) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
for (const entry of providerModels) {
|
||||
if (typeof entry === "string") {
|
||||
if (entry === modelID) {
|
||||
return { id: entry }
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if (entry?.id === modelID) {
|
||||
return entry
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
const defaultConnectedProvidersCacheStore = createConnectedProvidersCacheStore(
|
||||
() => dataPath.getOmoOpenCodeCacheDir()
|
||||
)
|
||||
|
||||
+17
-2
@@ -1,5 +1,18 @@
|
||||
import * as path from "node:path"
|
||||
import * as os from "node:os"
|
||||
import { accessSync, constants, mkdirSync } from "node:fs"
|
||||
|
||||
function resolveWritableDirectory(preferredDir: string, fallbackSuffix: string): string {
|
||||
try {
|
||||
mkdirSync(preferredDir, { recursive: true })
|
||||
accessSync(preferredDir, constants.W_OK)
|
||||
return preferredDir
|
||||
} catch {
|
||||
const fallbackDir = path.join(os.tmpdir(), fallbackSuffix)
|
||||
mkdirSync(fallbackDir, { recursive: true })
|
||||
return fallbackDir
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the user-level data directory.
|
||||
@@ -10,7 +23,8 @@ import * as os from "node:os"
|
||||
* including Windows, so we match that behavior exactly.
|
||||
*/
|
||||
export function getDataDir(): string {
|
||||
return process.env.XDG_DATA_HOME ?? path.join(os.homedir(), ".local", "share")
|
||||
const preferredDir = process.env.XDG_DATA_HOME ?? path.join(os.homedir(), ".local", "share")
|
||||
return resolveWritableDirectory(preferredDir, "opencode-data")
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -27,7 +41,8 @@ export function getOpenCodeStorageDir(): string {
|
||||
* - All platforms: XDG_CACHE_HOME or ~/.cache
|
||||
*/
|
||||
export function getCacheDir(): string {
|
||||
return process.env.XDG_CACHE_HOME ?? path.join(os.homedir(), ".cache")
|
||||
const preferredDir = process.env.XDG_CACHE_HOME ?? path.join(os.homedir(), ".cache")
|
||||
return resolveWritableDirectory(preferredDir, "opencode-cache")
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
import { describe, test, expect } from "bun:test"
|
||||
import { buildFallbackChainFromModels, parseFallbackModelEntry } from "./fallback-chain-from-models"
|
||||
import { describe, test, it, expect } from "bun:test"
|
||||
import {
|
||||
parseFallbackModelEntry,
|
||||
parseFallbackModelObjectEntry,
|
||||
buildFallbackChainFromModels,
|
||||
findMostSpecificFallbackEntry,
|
||||
} from "./fallback-chain-from-models"
|
||||
import { flattenToFallbackModelStrings } from "./model-resolver"
|
||||
|
||||
// Upstream tests
|
||||
describe("fallback-chain-from-models", () => {
|
||||
test("parses provider/model entry with parenthesized variant", () => {
|
||||
//#given
|
||||
@@ -61,3 +68,330 @@ describe("fallback-chain-from-models", () => {
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
// Object-style entry tests
|
||||
describe("parseFallbackModelEntry (extended)", () => {
|
||||
it("parses provider/model string", () => {
|
||||
const result = parseFallbackModelEntry("anthropic/claude-sonnet-4-6", undefined)
|
||||
expect(result).toEqual({
|
||||
providers: ["anthropic"],
|
||||
model: "claude-sonnet-4-6",
|
||||
})
|
||||
})
|
||||
|
||||
it("parses model with parenthesized variant", () => {
|
||||
const result = parseFallbackModelEntry("anthropic/claude-sonnet-4-6(high)", undefined)
|
||||
expect(result).toEqual({
|
||||
providers: ["anthropic"],
|
||||
model: "claude-sonnet-4-6",
|
||||
variant: "high",
|
||||
})
|
||||
})
|
||||
|
||||
it("parses model with space variant", () => {
|
||||
const result = parseFallbackModelEntry("openai/gpt-5.4 xhigh", undefined)
|
||||
expect(result).toEqual({
|
||||
providers: ["openai"],
|
||||
model: "gpt-5.4",
|
||||
variant: "xhigh",
|
||||
})
|
||||
})
|
||||
|
||||
it("parses model with minimal space variant", () => {
|
||||
const result = parseFallbackModelEntry("openai/gpt-5.4 minimal", undefined)
|
||||
expect(result).toEqual({
|
||||
providers: ["openai"],
|
||||
model: "gpt-5.4",
|
||||
variant: "minimal",
|
||||
})
|
||||
})
|
||||
|
||||
it("uses context provider when no provider prefix", () => {
|
||||
const result = parseFallbackModelEntry("claude-sonnet-4-6", "anthropic")
|
||||
expect(result).toEqual({
|
||||
providers: ["anthropic"],
|
||||
model: "claude-sonnet-4-6",
|
||||
})
|
||||
})
|
||||
|
||||
it("returns undefined for empty string", () => {
|
||||
expect(parseFallbackModelEntry("", undefined)).toBeUndefined()
|
||||
expect(parseFallbackModelEntry(" ", undefined)).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe("parseFallbackModelObjectEntry", () => {
|
||||
it("parses object with model only", () => {
|
||||
const result = parseFallbackModelObjectEntry(
|
||||
{ model: "anthropic/claude-sonnet-4-6" },
|
||||
undefined,
|
||||
)
|
||||
expect(result).toEqual({
|
||||
providers: ["anthropic"],
|
||||
model: "claude-sonnet-4-6",
|
||||
})
|
||||
})
|
||||
|
||||
it("parses object with variant override", () => {
|
||||
const result = parseFallbackModelObjectEntry(
|
||||
{ model: "anthropic/claude-sonnet-4-6", variant: "high" },
|
||||
undefined,
|
||||
)
|
||||
expect(result).toEqual({
|
||||
providers: ["anthropic"],
|
||||
model: "claude-sonnet-4-6",
|
||||
variant: "high",
|
||||
})
|
||||
})
|
||||
|
||||
it("object variant overrides inline variant", () => {
|
||||
const result = parseFallbackModelObjectEntry(
|
||||
{ model: "anthropic/claude-sonnet-4-6(low)", variant: "high" },
|
||||
undefined,
|
||||
)
|
||||
expect(result).toEqual({
|
||||
providers: ["anthropic"],
|
||||
model: "claude-sonnet-4-6",
|
||||
variant: "high",
|
||||
})
|
||||
})
|
||||
|
||||
it("carries reasoningEffort and temperature", () => {
|
||||
const result = parseFallbackModelObjectEntry(
|
||||
{
|
||||
model: "openai/gpt-5.4",
|
||||
variant: "high",
|
||||
reasoningEffort: "high",
|
||||
temperature: 0.5,
|
||||
},
|
||||
undefined,
|
||||
)
|
||||
expect(result).toEqual({
|
||||
providers: ["openai"],
|
||||
model: "gpt-5.4",
|
||||
variant: "high",
|
||||
reasoningEffort: "high",
|
||||
temperature: 0.5,
|
||||
})
|
||||
})
|
||||
|
||||
it("carries thinking config", () => {
|
||||
const result = parseFallbackModelObjectEntry(
|
||||
{
|
||||
model: "anthropic/claude-sonnet-4-6",
|
||||
thinking: { type: "enabled", budgetTokens: 10000 },
|
||||
},
|
||||
undefined,
|
||||
)
|
||||
expect(result).toEqual({
|
||||
providers: ["anthropic"],
|
||||
model: "claude-sonnet-4-6",
|
||||
thinking: { type: "enabled", budgetTokens: 10000 },
|
||||
})
|
||||
})
|
||||
|
||||
it("carries all optional fields", () => {
|
||||
const result = parseFallbackModelObjectEntry(
|
||||
{
|
||||
model: "openai/gpt-5.4",
|
||||
variant: "xhigh",
|
||||
reasoningEffort: "xhigh",
|
||||
temperature: 0.3,
|
||||
top_p: 0.9,
|
||||
maxTokens: 8192,
|
||||
thinking: { type: "disabled" },
|
||||
},
|
||||
undefined,
|
||||
)
|
||||
expect(result).toEqual({
|
||||
providers: ["openai"],
|
||||
model: "gpt-5.4",
|
||||
variant: "xhigh",
|
||||
reasoningEffort: "xhigh",
|
||||
temperature: 0.3,
|
||||
top_p: 0.9,
|
||||
maxTokens: 8192,
|
||||
thinking: { type: "disabled" },
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe("buildFallbackChainFromModels (mixed)", () => {
|
||||
it("handles string input", () => {
|
||||
const result = buildFallbackChainFromModels("anthropic/claude-sonnet-4-6", undefined)
|
||||
expect(result).toEqual([
|
||||
{ providers: ["anthropic"], model: "claude-sonnet-4-6" },
|
||||
])
|
||||
})
|
||||
|
||||
it("handles string array", () => {
|
||||
const result = buildFallbackChainFromModels(
|
||||
["anthropic/claude-sonnet-4-6", "openai/gpt-5.4"],
|
||||
undefined,
|
||||
)
|
||||
expect(result).toEqual([
|
||||
{ providers: ["anthropic"], model: "claude-sonnet-4-6" },
|
||||
{ providers: ["openai"], model: "gpt-5.4" },
|
||||
])
|
||||
})
|
||||
|
||||
it("handles mixed array of strings and objects", () => {
|
||||
const result = buildFallbackChainFromModels(
|
||||
[
|
||||
{ model: "anthropic/claude-sonnet-4-6", variant: "high", reasoningEffort: "high" },
|
||||
{ model: "openai/gpt-5.4", reasoningEffort: "xhigh" },
|
||||
"chutes/kimi-k2.5",
|
||||
{ model: "chutes/glm-5", temperature: 0.7 },
|
||||
"google/gemini-3-flash",
|
||||
],
|
||||
undefined,
|
||||
)
|
||||
expect(result).toEqual([
|
||||
{ providers: ["anthropic"], model: "claude-sonnet-4-6", variant: "high", reasoningEffort: "high" },
|
||||
{ providers: ["openai"], model: "gpt-5.4", reasoningEffort: "xhigh" },
|
||||
{ providers: ["chutes"], model: "kimi-k2.5" },
|
||||
{ providers: ["chutes"], model: "glm-5", temperature: 0.7 },
|
||||
{ providers: ["google"], model: "gemini-3-flash" },
|
||||
])
|
||||
})
|
||||
|
||||
it("returns undefined for empty/undefined input", () => {
|
||||
expect(buildFallbackChainFromModels(undefined, undefined)).toBeUndefined()
|
||||
expect(buildFallbackChainFromModels([], undefined)).toBeUndefined()
|
||||
})
|
||||
|
||||
it("filters out invalid entries", () => {
|
||||
const result = buildFallbackChainFromModels(
|
||||
["", "anthropic/claude-sonnet-4-6", " "],
|
||||
undefined,
|
||||
)
|
||||
expect(result).toEqual([
|
||||
{ providers: ["anthropic"], model: "claude-sonnet-4-6" },
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe("flattenToFallbackModelStrings", () => {
|
||||
it("returns undefined for undefined input", () => {
|
||||
expect(flattenToFallbackModelStrings(undefined)).toBeUndefined()
|
||||
})
|
||||
|
||||
it("passes through plain strings", () => {
|
||||
expect(flattenToFallbackModelStrings(["anthropic/claude-sonnet-4-6"])).toEqual([
|
||||
"anthropic/claude-sonnet-4-6",
|
||||
])
|
||||
})
|
||||
|
||||
it("flattens object with explicit variant", () => {
|
||||
expect(flattenToFallbackModelStrings([
|
||||
{ model: "anthropic/claude-sonnet-4-6", variant: "high" },
|
||||
])).toEqual(["anthropic/claude-sonnet-4-6(high)"])
|
||||
})
|
||||
|
||||
it("preserves inline variant when no explicit variant", () => {
|
||||
expect(flattenToFallbackModelStrings([
|
||||
{ model: "anthropic/claude-sonnet-4-6(high)" },
|
||||
])).toEqual(["anthropic/claude-sonnet-4-6(high)"])
|
||||
})
|
||||
|
||||
it("explicit variant overrides inline variant (no double-suffix)", () => {
|
||||
expect(flattenToFallbackModelStrings([
|
||||
{ model: "anthropic/claude-sonnet-4-6(low)", variant: "high" },
|
||||
])).toEqual(["anthropic/claude-sonnet-4-6(high)"])
|
||||
})
|
||||
|
||||
it("explicit variant overrides space-suffix variant", () => {
|
||||
expect(flattenToFallbackModelStrings([
|
||||
{ model: "openai/gpt-5.4 high", variant: "low" },
|
||||
])).toEqual(["openai/gpt-5.4(low)"])
|
||||
})
|
||||
|
||||
it("explicit variant overrides minimal space-suffix variant", () => {
|
||||
expect(flattenToFallbackModelStrings([
|
||||
{ model: "openai/gpt-5.4 minimal", variant: "low" },
|
||||
])).toEqual(["openai/gpt-5.4(low)"])
|
||||
})
|
||||
|
||||
it("preserves trailing non-variant suffixes when adding explicit variant", () => {
|
||||
expect(flattenToFallbackModelStrings([
|
||||
{ model: "openai/gpt-5.4 preview", variant: "low" },
|
||||
])).toEqual(["openai/gpt-5.4 preview(low)"])
|
||||
})
|
||||
|
||||
it("flattens object without variant", () => {
|
||||
expect(flattenToFallbackModelStrings([
|
||||
{ model: "openai/gpt-5.4" },
|
||||
])).toEqual(["openai/gpt-5.4"])
|
||||
})
|
||||
|
||||
it("handles mixed array", () => {
|
||||
expect(flattenToFallbackModelStrings([
|
||||
"anthropic/claude-sonnet-4-6",
|
||||
{ model: "openai/gpt-5.4", variant: "high" },
|
||||
{ model: "google/gemini-3-flash(low)" },
|
||||
])).toEqual([
|
||||
"anthropic/claude-sonnet-4-6",
|
||||
"openai/gpt-5.4(high)",
|
||||
"google/gemini-3-flash(low)",
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe("findMostSpecificFallbackEntry", () => {
|
||||
it("picks exact match over prefix match", () => {
|
||||
const chain = [
|
||||
{ providers: ["openai"], model: "gpt-5.4" },
|
||||
{ providers: ["openai"], model: "gpt-5.4-preview" },
|
||||
]
|
||||
const result = findMostSpecificFallbackEntry("openai", "gpt-5.4-preview", chain)
|
||||
expect(result?.model).toBe("gpt-5.4-preview")
|
||||
})
|
||||
|
||||
it("returns prefix match when no exact match exists", () => {
|
||||
const chain = [
|
||||
{ providers: ["openai"], model: "gpt-5.4" },
|
||||
]
|
||||
const result = findMostSpecificFallbackEntry("openai", "gpt-5.4-preview", chain)
|
||||
expect(result?.model).toBe("gpt-5.4")
|
||||
})
|
||||
|
||||
it("returns undefined when no entry matches", () => {
|
||||
const chain = [
|
||||
{ providers: ["anthropic"], model: "claude-sonnet-4-6" },
|
||||
]
|
||||
expect(findMostSpecificFallbackEntry("openai", "gpt-5.4", chain)).toBeUndefined()
|
||||
})
|
||||
|
||||
it("sorts by matched prefix length, not insertion order", () => {
|
||||
// Both entries share the same provider so both match as prefixes;
|
||||
// the longer (more-specific) prefix must win regardless of array order.
|
||||
const chain = [
|
||||
{ providers: ["openai"], model: "gpt-5" },
|
||||
{ providers: ["openai"], model: "gpt-5.4-preview" },
|
||||
]
|
||||
const result = findMostSpecificFallbackEntry("openai", "gpt-5.4-preview-2026", chain)
|
||||
expect(result?.model).toBe("gpt-5.4-preview")
|
||||
})
|
||||
|
||||
it("is case-insensitive", () => {
|
||||
const chain = [
|
||||
{ providers: ["OpenAI"], model: "GPT-5.4" },
|
||||
]
|
||||
const result = findMostSpecificFallbackEntry("openai", "gpt-5.4-preview", chain)
|
||||
expect(result?.model).toBe("GPT-5.4")
|
||||
})
|
||||
|
||||
it("preserves variant and settings from matched entry", () => {
|
||||
const chain = [
|
||||
{ providers: ["openai"], model: "gpt-5.4", variant: "high", temperature: 0.7 },
|
||||
{ providers: ["openai"], model: "gpt-5.4-preview", variant: "low", reasoningEffort: "medium" },
|
||||
]
|
||||
const result = findMostSpecificFallbackEntry("openai", "gpt-5.4-preview", chain)
|
||||
expect(result).toEqual({
|
||||
providers: ["openai"],
|
||||
model: "gpt-5.4-preview",
|
||||
variant: "low",
|
||||
reasoningEffort: "medium",
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,16 +1,7 @@
|
||||
import type { FallbackEntry } from "./model-requirements"
|
||||
import type { FallbackModelObject } from "../config/schema/fallback-models"
|
||||
import { normalizeFallbackModels } from "./model-resolver"
|
||||
|
||||
const KNOWN_VARIANTS = new Set([
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max",
|
||||
"none",
|
||||
"auto",
|
||||
"thinking",
|
||||
])
|
||||
import { KNOWN_VARIANTS } from "./known-variants"
|
||||
|
||||
function parseVariantFromModel(rawModel: string): { modelID: string; variant?: string } {
|
||||
const trimmedModel = rawModel.trim()
|
||||
@@ -61,8 +52,58 @@ export function parseFallbackModelEntry(
|
||||
}
|
||||
}
|
||||
|
||||
export function parseFallbackModelObjectEntry(
|
||||
obj: FallbackModelObject,
|
||||
contextProviderID: string | undefined,
|
||||
defaultProviderID = "opencode",
|
||||
): FallbackEntry | undefined {
|
||||
const base = parseFallbackModelEntry(obj.model, contextProviderID, defaultProviderID)
|
||||
if (!base) return undefined
|
||||
|
||||
return {
|
||||
...base,
|
||||
variant: obj.variant ?? base.variant,
|
||||
reasoningEffort: obj.reasoningEffort,
|
||||
temperature: obj.temperature,
|
||||
top_p: obj.top_p,
|
||||
maxTokens: obj.maxTokens,
|
||||
thinking: obj.thinking,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the most specific FallbackEntry whose `provider/model` is a prefix of
|
||||
* the resolved `provider/modelID`. Longest match wins so that e.g.
|
||||
* `openai/gpt-5.4-preview` picks the entry for `openai/gpt-5.4-preview` over
|
||||
* the shorter `openai/gpt-5.4`.
|
||||
*/
|
||||
export function findMostSpecificFallbackEntry(
|
||||
providerID: string,
|
||||
modelID: string,
|
||||
chain: FallbackEntry[],
|
||||
): FallbackEntry | undefined {
|
||||
const resolved = `${providerID}/${modelID}`.toLowerCase()
|
||||
|
||||
// Collect entries whose provider/model is a prefix of the resolved model,
|
||||
// together with the length of the matching prefix (longest match wins).
|
||||
const matches: { entry: FallbackEntry; matchLen: number }[] = []
|
||||
for (const entry of chain) {
|
||||
for (const p of entry.providers) {
|
||||
const candidate = `${p}/${entry.model}`.toLowerCase()
|
||||
if (resolved.startsWith(candidate)) {
|
||||
matches.push({ entry, matchLen: candidate.length })
|
||||
break // one match per entry is enough
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (matches.length === 0) return undefined
|
||||
matches.sort((a, b) => b.matchLen - a.matchLen)
|
||||
return matches[0].entry
|
||||
}
|
||||
|
||||
export function buildFallbackChainFromModels(
|
||||
fallbackModels: string | string[] | undefined,
|
||||
fallbackModels: string | (string | FallbackModelObject)[] | undefined,
|
||||
contextProviderID: string | undefined,
|
||||
defaultProviderID = "opencode",
|
||||
): FallbackEntry[] | undefined {
|
||||
@@ -70,7 +111,12 @@ export function buildFallbackChainFromModels(
|
||||
if (!normalized || normalized.length === 0) return undefined
|
||||
|
||||
const parsed = normalized
|
||||
.map((model) => parseFallbackModelEntry(model, contextProviderID, defaultProviderID))
|
||||
.map((entry) => {
|
||||
if (typeof entry === "string") {
|
||||
return parseFallbackModelEntry(entry, contextProviderID, defaultProviderID)
|
||||
}
|
||||
return parseFallbackModelObjectEntry(entry, contextProviderID, defaultProviderID)
|
||||
})
|
||||
.filter((entry): entry is FallbackEntry => entry !== undefined)
|
||||
|
||||
if (parsed.length === 0) return undefined
|
||||
|
||||
+5
-1
@@ -35,7 +35,7 @@ export * from "./agent-tool-restrictions"
|
||||
export * from "./model-requirements"
|
||||
export * from "./model-resolver"
|
||||
export { normalizeModel, normalizeModelID } from "./model-normalization"
|
||||
export { normalizeFallbackModels } from "./model-resolver"
|
||||
export { normalizeFallbackModels, flattenToFallbackModelStrings } from "./model-resolver"
|
||||
export { resolveModelPipeline } from "./model-resolution-pipeline"
|
||||
export type {
|
||||
ModelResolutionRequest,
|
||||
@@ -43,6 +43,10 @@ export type {
|
||||
ModelResolutionResult,
|
||||
} from "./model-resolution-types"
|
||||
export * from "./model-availability"
|
||||
export * from "./model-capabilities"
|
||||
export * from "./model-capabilities-cache"
|
||||
export * from "./model-capability-heuristics"
|
||||
export * from "./model-settings-compatibility"
|
||||
export * from "./fallback-model-availability"
|
||||
export * from "./connected-providers-cache"
|
||||
export * from "./context-limit-resolver"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { detectConfigFile, parseJsonc, parseJsoncSafe, readJsoncFile } from "./jsonc-parser"
|
||||
import { detectConfigFile, detectPluginConfigFile, parseJsonc, parseJsoncSafe, readJsoncFile } from "./jsonc-parser"
|
||||
import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"
|
||||
import { join } from "node:path"
|
||||
|
||||
@@ -264,3 +264,84 @@ describe("detectConfigFile", () => {
|
||||
expect(result.format).toBe("none")
|
||||
})
|
||||
})
|
||||
|
||||
describe("detectPluginConfigFile", () => {
|
||||
const testDir = join(__dirname, ".test-detect-plugin")
|
||||
|
||||
test("prefers oh-my-opencode over oh-my-openagent", () => {
|
||||
// given
|
||||
if (!existsSync(testDir)) mkdirSync(testDir, { recursive: true })
|
||||
writeFileSync(join(testDir, "oh-my-openagent.jsonc"), "{}")
|
||||
writeFileSync(join(testDir, "oh-my-opencode.jsonc"), "{}")
|
||||
|
||||
// when
|
||||
const result = detectPluginConfigFile(testDir)
|
||||
|
||||
// then
|
||||
expect(result.format).toBe("jsonc")
|
||||
expect(result.path).toBe(join(testDir, "oh-my-opencode.jsonc"))
|
||||
|
||||
rmSync(testDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
test("falls back to oh-my-opencode when oh-my-openagent doesn't exist", () => {
|
||||
// given
|
||||
if (!existsSync(testDir)) mkdirSync(testDir, { recursive: true })
|
||||
writeFileSync(join(testDir, "oh-my-opencode.jsonc"), "{}")
|
||||
|
||||
// when
|
||||
const result = detectPluginConfigFile(testDir)
|
||||
|
||||
// then
|
||||
expect(result.format).toBe("jsonc")
|
||||
expect(result.path).toBe(join(testDir, "oh-my-opencode.jsonc"))
|
||||
|
||||
rmSync(testDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
test("falls back to oh-my-opencode.json when no jsonc exists", () => {
|
||||
// given
|
||||
if (!existsSync(testDir)) mkdirSync(testDir, { recursive: true })
|
||||
writeFileSync(join(testDir, "oh-my-opencode.json"), "{}")
|
||||
|
||||
// when
|
||||
const result = detectPluginConfigFile(testDir)
|
||||
|
||||
// then
|
||||
expect(result.format).toBe("json")
|
||||
expect(result.path).toBe(join(testDir, "oh-my-opencode.json"))
|
||||
|
||||
rmSync(testDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
test("returns none when no config files exist", () => {
|
||||
// given
|
||||
const emptyDir = join(testDir, "empty")
|
||||
if (!existsSync(emptyDir)) mkdirSync(emptyDir, { recursive: true })
|
||||
|
||||
// when
|
||||
const result = detectPluginConfigFile(emptyDir)
|
||||
|
||||
// then
|
||||
expect(result.format).toBe("none")
|
||||
expect(result.path).toBe(join(emptyDir, "oh-my-opencode.json"))
|
||||
|
||||
rmSync(testDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
test("prefers oh-my-opencode.json over oh-my-openagent.jsonc", () => {
|
||||
// given
|
||||
if (!existsSync(testDir)) mkdirSync(testDir, { recursive: true })
|
||||
writeFileSync(join(testDir, "oh-my-opencode.json"), "{}")
|
||||
writeFileSync(join(testDir, "oh-my-openagent.jsonc"), "{}")
|
||||
|
||||
// when
|
||||
const result = detectPluginConfigFile(testDir)
|
||||
|
||||
// then
|
||||
expect(result.format).toBe("json")
|
||||
expect(result.path).toBe(join(testDir, "oh-my-opencode.json"))
|
||||
|
||||
rmSync(testDir, { recursive: true, force: true })
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { existsSync, readFileSync } from "node:fs"
|
||||
import { join } from "node:path"
|
||||
import { parse, ParseError, printParseErrorCode } from "jsonc-parser"
|
||||
|
||||
export interface JsoncParseResult<T> {
|
||||
@@ -64,3 +65,16 @@ export function detectConfigFile(basePath: string): {
|
||||
}
|
||||
return { format: "none", path: jsonPath }
|
||||
}
|
||||
|
||||
const PLUGIN_CONFIG_NAMES = ["oh-my-opencode", "oh-my-openagent"] as const
|
||||
|
||||
export function detectPluginConfigFile(dir: string): {
|
||||
format: "json" | "jsonc" | "none"
|
||||
path: string
|
||||
} {
|
||||
for (const name of PLUGIN_CONFIG_NAMES) {
|
||||
const result = detectConfigFile(join(dir, name))
|
||||
if (result.format !== "none") return result
|
||||
}
|
||||
return { format: "none", path: join(dir, PLUGIN_CONFIG_NAMES[0] + ".json") }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
/**
|
||||
* Canonical set of recognised variant / effort tokens.
|
||||
* Used by parseFallbackModelEntry (space-suffix detection) and
|
||||
* flattenToFallbackModelStrings (inline-variant stripping).
|
||||
*/
|
||||
export const KNOWN_VARIANTS = new Set([
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max",
|
||||
"minimal",
|
||||
"none",
|
||||
"auto",
|
||||
"thinking",
|
||||
])
|
||||
@@ -0,0 +1,165 @@
|
||||
/// <reference types="bun-types" />
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, test } from "bun:test"
|
||||
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"
|
||||
import { tmpdir } from "node:os"
|
||||
import { join } from "node:path"
|
||||
import {
|
||||
buildModelCapabilitiesSnapshotFromModelsDev,
|
||||
createModelCapabilitiesCacheStore,
|
||||
MODELS_DEV_SOURCE_URL,
|
||||
} from "./model-capabilities-cache"
|
||||
|
||||
let fakeUserCacheRoot = ""
|
||||
let testCacheDir = ""
|
||||
|
||||
describe("model-capabilities-cache", () => {
|
||||
beforeEach(() => {
|
||||
fakeUserCacheRoot = mkdtempSync(join(tmpdir(), "model-capabilities-cache-"))
|
||||
testCacheDir = join(fakeUserCacheRoot, "oh-my-opencode")
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(fakeUserCacheRoot)) {
|
||||
rmSync(fakeUserCacheRoot, { recursive: true, force: true })
|
||||
}
|
||||
fakeUserCacheRoot = ""
|
||||
testCacheDir = ""
|
||||
})
|
||||
|
||||
test("builds a normalized snapshot from provider-keyed models.dev data", () => {
|
||||
//#given
|
||||
const raw = {
|
||||
openai: {
|
||||
models: {
|
||||
"gpt-5.4": {
|
||||
id: "gpt-5.4",
|
||||
family: "gpt",
|
||||
reasoning: true,
|
||||
temperature: false,
|
||||
tool_call: true,
|
||||
modalities: {
|
||||
input: ["text", "image"],
|
||||
output: ["text"],
|
||||
},
|
||||
limit: {
|
||||
context: 1_050_000,
|
||||
output: 128_000,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
anthropic: {
|
||||
models: {
|
||||
"claude-sonnet-4-6": {
|
||||
family: "claude-sonnet",
|
||||
reasoning: true,
|
||||
temperature: true,
|
||||
limit: {
|
||||
context: 1_000_000,
|
||||
output: 64_000,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
//#when
|
||||
const snapshot = buildModelCapabilitiesSnapshotFromModelsDev(raw)
|
||||
|
||||
//#then
|
||||
expect(snapshot.sourceUrl).toBe(MODELS_DEV_SOURCE_URL)
|
||||
expect(snapshot.models["gpt-5.4"]).toEqual({
|
||||
id: "gpt-5.4",
|
||||
family: "gpt",
|
||||
reasoning: true,
|
||||
temperature: false,
|
||||
toolCall: true,
|
||||
modalities: {
|
||||
input: ["text", "image"],
|
||||
output: ["text"],
|
||||
},
|
||||
limit: {
|
||||
context: 1_050_000,
|
||||
output: 128_000,
|
||||
},
|
||||
})
|
||||
expect(snapshot.models["claude-sonnet-4-6"]).toEqual({
|
||||
id: "claude-sonnet-4-6",
|
||||
family: "claude-sonnet",
|
||||
reasoning: true,
|
||||
temperature: true,
|
||||
limit: {
|
||||
context: 1_000_000,
|
||||
output: 64_000,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
test("merges repeated snapshot entries without materializing empty optional objects", () => {
|
||||
const raw = {
|
||||
openai: {
|
||||
models: {
|
||||
"gpt-5.4": {
|
||||
id: "gpt-5.4",
|
||||
family: "gpt",
|
||||
},
|
||||
},
|
||||
},
|
||||
alias: {
|
||||
models: {
|
||||
"gpt-5.4-preview": {
|
||||
id: "gpt-5.4",
|
||||
reasoning: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
const snapshot = buildModelCapabilitiesSnapshotFromModelsDev(raw)
|
||||
|
||||
expect(snapshot.models["gpt-5.4"]).toEqual({
|
||||
id: "gpt-5.4",
|
||||
family: "gpt",
|
||||
reasoning: true,
|
||||
})
|
||||
expect(snapshot.models["gpt-5.4"]).not.toHaveProperty("modalities")
|
||||
expect(snapshot.models["gpt-5.4"]).not.toHaveProperty("limit")
|
||||
})
|
||||
|
||||
test("refresh writes cache and preserves unrelated files in the cache directory", async () => {
|
||||
//#given
|
||||
const sentinelPath = join(testCacheDir, "keep-me.json")
|
||||
const store = createModelCapabilitiesCacheStore(() => testCacheDir)
|
||||
mkdirSync(testCacheDir, { recursive: true })
|
||||
writeFileSync(sentinelPath, JSON.stringify({ keep: true }))
|
||||
|
||||
const fetchImpl: typeof fetch = async () =>
|
||||
new Response(JSON.stringify({
|
||||
openai: {
|
||||
models: {
|
||||
"gpt-5.4": {
|
||||
id: "gpt-5.4",
|
||||
family: "gpt",
|
||||
reasoning: true,
|
||||
limit: { output: 128_000 },
|
||||
},
|
||||
},
|
||||
},
|
||||
}), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
|
||||
//#when
|
||||
const snapshot = await store.refreshModelCapabilitiesCache({ fetchImpl })
|
||||
const reloadedStore = createModelCapabilitiesCacheStore(() => testCacheDir)
|
||||
|
||||
//#then
|
||||
expect(snapshot.models["gpt-5.4"]?.limit?.output).toBe(128_000)
|
||||
expect(existsSync(sentinelPath)).toBe(true)
|
||||
expect(readFileSync(sentinelPath, "utf-8")).toBe(JSON.stringify({ keep: true }))
|
||||
expect(reloadedStore.readModelCapabilitiesCache()).toEqual(snapshot)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,248 @@
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"
|
||||
import { join } from "path"
|
||||
import * as dataPath from "./data-path"
|
||||
import { log } from "./logger"
|
||||
import type { ModelCapabilitiesSnapshot, ModelCapabilitiesSnapshotEntry } from "./model-capabilities"
|
||||
|
||||
export const MODELS_DEV_SOURCE_URL = "https://models.dev/api.json"
|
||||
const MODEL_CAPABILITIES_CACHE_FILE = "model-capabilities.json"
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
}
|
||||
|
||||
function readBoolean(value: unknown): boolean | undefined {
|
||||
return typeof value === "boolean" ? value : undefined
|
||||
}
|
||||
|
||||
function readNumber(value: unknown): number | undefined {
|
||||
return typeof value === "number" ? value : undefined
|
||||
}
|
||||
|
||||
function readString(value: unknown): string | undefined {
|
||||
return typeof value === "string" ? value : undefined
|
||||
}
|
||||
|
||||
function readStringArray(value: unknown): string[] | undefined {
|
||||
if (!Array.isArray(value)) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const result = value.filter((item): item is string => typeof item === "string")
|
||||
return result.length > 0 ? result : undefined
|
||||
}
|
||||
|
||||
function normalizeSnapshotEntry(rawModelID: string, rawModel: unknown): ModelCapabilitiesSnapshotEntry | undefined {
|
||||
if (!isRecord(rawModel)) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const id = readString(rawModel.id) ?? rawModelID
|
||||
const family = readString(rawModel.family)
|
||||
const reasoning = readBoolean(rawModel.reasoning)
|
||||
const temperature = readBoolean(rawModel.temperature)
|
||||
const toolCall = readBoolean(rawModel.tool_call)
|
||||
|
||||
const rawModalities = isRecord(rawModel.modalities) ? rawModel.modalities : undefined
|
||||
const modalitiesInput = readStringArray(rawModalities?.input)
|
||||
const modalitiesOutput = readStringArray(rawModalities?.output)
|
||||
const modalities = modalitiesInput || modalitiesOutput
|
||||
? {
|
||||
...(modalitiesInput ? { input: modalitiesInput } : {}),
|
||||
...(modalitiesOutput ? { output: modalitiesOutput } : {}),
|
||||
}
|
||||
: undefined
|
||||
|
||||
const rawLimit = isRecord(rawModel.limit) ? rawModel.limit : undefined
|
||||
const limitContext = readNumber(rawLimit?.context)
|
||||
const limitInput = readNumber(rawLimit?.input)
|
||||
const limitOutput = readNumber(rawLimit?.output)
|
||||
const limit = limitContext !== undefined || limitInput !== undefined || limitOutput !== undefined
|
||||
? {
|
||||
...(limitContext !== undefined ? { context: limitContext } : {}),
|
||||
...(limitInput !== undefined ? { input: limitInput } : {}),
|
||||
...(limitOutput !== undefined ? { output: limitOutput } : {}),
|
||||
}
|
||||
: undefined
|
||||
|
||||
return {
|
||||
id,
|
||||
...(family ? { family } : {}),
|
||||
...(reasoning !== undefined ? { reasoning } : {}),
|
||||
...(temperature !== undefined ? { temperature } : {}),
|
||||
...(toolCall !== undefined ? { toolCall } : {}),
|
||||
...(modalities ? { modalities } : {}),
|
||||
...(limit ? { limit } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
function mergeSnapshotEntries(
|
||||
existing: ModelCapabilitiesSnapshotEntry | undefined,
|
||||
incoming: ModelCapabilitiesSnapshotEntry,
|
||||
): ModelCapabilitiesSnapshotEntry {
|
||||
if (!existing) {
|
||||
return incoming
|
||||
}
|
||||
|
||||
const mergedModalities = existing.modalities || incoming.modalities
|
||||
? {
|
||||
...existing.modalities,
|
||||
...incoming.modalities,
|
||||
}
|
||||
: undefined
|
||||
const mergedLimit = existing.limit || incoming.limit
|
||||
? {
|
||||
...existing.limit,
|
||||
...incoming.limit,
|
||||
}
|
||||
: undefined
|
||||
|
||||
return {
|
||||
...existing,
|
||||
...incoming,
|
||||
...(mergedModalities ? { modalities: mergedModalities } : {}),
|
||||
...(mergedLimit ? { limit: mergedLimit } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
export function buildModelCapabilitiesSnapshotFromModelsDev(raw: unknown): ModelCapabilitiesSnapshot {
|
||||
const models: Record<string, ModelCapabilitiesSnapshotEntry> = {}
|
||||
const providers = isRecord(raw) ? raw : {}
|
||||
|
||||
for (const providerValue of Object.values(providers)) {
|
||||
if (!isRecord(providerValue)) {
|
||||
continue
|
||||
}
|
||||
|
||||
const providerModels = providerValue.models
|
||||
if (!isRecord(providerModels)) {
|
||||
continue
|
||||
}
|
||||
|
||||
for (const [rawModelID, rawModel] of Object.entries(providerModels)) {
|
||||
const normalizedEntry = normalizeSnapshotEntry(rawModelID, rawModel)
|
||||
if (!normalizedEntry) {
|
||||
continue
|
||||
}
|
||||
|
||||
models[normalizedEntry.id.toLowerCase()] = mergeSnapshotEntries(
|
||||
models[normalizedEntry.id.toLowerCase()],
|
||||
normalizedEntry,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
generatedAt: new Date().toISOString(),
|
||||
sourceUrl: MODELS_DEV_SOURCE_URL,
|
||||
models,
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchModelCapabilitiesSnapshot(args: {
|
||||
sourceUrl?: string
|
||||
fetchImpl?: typeof fetch
|
||||
} = {}): Promise<ModelCapabilitiesSnapshot> {
|
||||
const sourceUrl = args.sourceUrl ?? MODELS_DEV_SOURCE_URL
|
||||
const fetchImpl = args.fetchImpl ?? fetch
|
||||
const response = await fetchImpl(sourceUrl)
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`models.dev fetch failed with ${response.status}`)
|
||||
}
|
||||
|
||||
const raw = await response.json()
|
||||
const snapshot = buildModelCapabilitiesSnapshotFromModelsDev(raw)
|
||||
return {
|
||||
...snapshot,
|
||||
sourceUrl,
|
||||
}
|
||||
}
|
||||
|
||||
export function createModelCapabilitiesCacheStore(
|
||||
getCacheDir: () => string = dataPath.getOmoOpenCodeCacheDir,
|
||||
) {
|
||||
let memSnapshot: ModelCapabilitiesSnapshot | null | undefined
|
||||
|
||||
function getCacheFilePath(): string {
|
||||
return join(getCacheDir(), MODEL_CAPABILITIES_CACHE_FILE)
|
||||
}
|
||||
|
||||
function ensureCacheDir(): void {
|
||||
const cacheDir = getCacheDir()
|
||||
if (!existsSync(cacheDir)) {
|
||||
mkdirSync(cacheDir, { recursive: true })
|
||||
}
|
||||
}
|
||||
|
||||
function readModelCapabilitiesCache(): ModelCapabilitiesSnapshot | null {
|
||||
if (memSnapshot !== undefined) {
|
||||
return memSnapshot
|
||||
}
|
||||
|
||||
const cacheFile = getCacheFilePath()
|
||||
if (!existsSync(cacheFile)) {
|
||||
memSnapshot = null
|
||||
log("[model-capabilities-cache] Cache file not found", { cacheFile })
|
||||
return null
|
||||
}
|
||||
|
||||
try {
|
||||
const content = readFileSync(cacheFile, "utf-8")
|
||||
const snapshot = JSON.parse(content) as ModelCapabilitiesSnapshot
|
||||
memSnapshot = snapshot
|
||||
log("[model-capabilities-cache] Read cache", {
|
||||
modelCount: Object.keys(snapshot.models).length,
|
||||
generatedAt: snapshot.generatedAt,
|
||||
})
|
||||
return snapshot
|
||||
} catch (error) {
|
||||
memSnapshot = null
|
||||
log("[model-capabilities-cache] Error reading cache", { error: String(error) })
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function hasModelCapabilitiesCache(): boolean {
|
||||
return existsSync(getCacheFilePath())
|
||||
}
|
||||
|
||||
function writeModelCapabilitiesCache(snapshot: ModelCapabilitiesSnapshot): void {
|
||||
ensureCacheDir()
|
||||
const cacheFile = getCacheFilePath()
|
||||
|
||||
writeFileSync(cacheFile, JSON.stringify(snapshot, null, 2) + "\n")
|
||||
memSnapshot = snapshot
|
||||
log("[model-capabilities-cache] Cache written", {
|
||||
modelCount: Object.keys(snapshot.models).length,
|
||||
generatedAt: snapshot.generatedAt,
|
||||
})
|
||||
}
|
||||
|
||||
async function refreshModelCapabilitiesCache(args: {
|
||||
sourceUrl?: string
|
||||
fetchImpl?: typeof fetch
|
||||
} = {}): Promise<ModelCapabilitiesSnapshot> {
|
||||
const snapshot = await fetchModelCapabilitiesSnapshot(args)
|
||||
writeModelCapabilitiesCache(snapshot)
|
||||
return snapshot
|
||||
}
|
||||
|
||||
return {
|
||||
readModelCapabilitiesCache,
|
||||
hasModelCapabilitiesCache,
|
||||
writeModelCapabilitiesCache,
|
||||
refreshModelCapabilitiesCache,
|
||||
}
|
||||
}
|
||||
|
||||
const defaultModelCapabilitiesCacheStore = createModelCapabilitiesCacheStore(
|
||||
() => dataPath.getOmoOpenCodeCacheDir(),
|
||||
)
|
||||
|
||||
export const {
|
||||
readModelCapabilitiesCache,
|
||||
hasModelCapabilitiesCache,
|
||||
writeModelCapabilitiesCache,
|
||||
refreshModelCapabilitiesCache,
|
||||
} = defaultModelCapabilitiesCacheStore
|
||||
@@ -0,0 +1,310 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
|
||||
import {
|
||||
getModelCapabilities,
|
||||
getBundledModelCapabilitiesSnapshot,
|
||||
type ModelCapabilitiesSnapshot,
|
||||
} from "./model-capabilities"
|
||||
import { AGENT_MODEL_REQUIREMENTS, CATEGORY_MODEL_REQUIREMENTS } from "./model-requirements"
|
||||
|
||||
describe("getModelCapabilities", () => {
|
||||
const bundledSnapshot: ModelCapabilitiesSnapshot = {
|
||||
generatedAt: "2026-03-25T00:00:00.000Z",
|
||||
sourceUrl: "https://models.dev/api.json",
|
||||
models: {
|
||||
"claude-opus-4-6": {
|
||||
id: "claude-opus-4-6",
|
||||
family: "claude-opus",
|
||||
reasoning: true,
|
||||
temperature: true,
|
||||
modalities: {
|
||||
input: ["text", "image", "pdf"],
|
||||
output: ["text"],
|
||||
},
|
||||
limit: {
|
||||
context: 1_000_000,
|
||||
output: 128_000,
|
||||
},
|
||||
toolCall: true,
|
||||
},
|
||||
"gemini-3.1-pro": {
|
||||
id: "gemini-3.1-pro",
|
||||
family: "gemini",
|
||||
reasoning: true,
|
||||
temperature: true,
|
||||
modalities: {
|
||||
input: ["text", "image"],
|
||||
output: ["text"],
|
||||
},
|
||||
limit: {
|
||||
context: 1_000_000,
|
||||
output: 65_000,
|
||||
},
|
||||
},
|
||||
"gpt-5.4": {
|
||||
id: "gpt-5.4",
|
||||
family: "gpt",
|
||||
reasoning: true,
|
||||
temperature: false,
|
||||
modalities: {
|
||||
input: ["text", "image", "pdf"],
|
||||
output: ["text"],
|
||||
},
|
||||
limit: {
|
||||
context: 1_050_000,
|
||||
output: 128_000,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
test("uses runtime metadata before snapshot data", () => {
|
||||
const result = getModelCapabilities({
|
||||
providerID: "anthropic",
|
||||
modelID: "claude-opus-4-6",
|
||||
runtimeModel: {
|
||||
variants: {
|
||||
low: {},
|
||||
medium: {},
|
||||
high: {},
|
||||
},
|
||||
},
|
||||
bundledSnapshot,
|
||||
})
|
||||
|
||||
expect(result).toMatchObject({
|
||||
canonicalModelID: "claude-opus-4-6",
|
||||
family: "claude-opus",
|
||||
variants: ["low", "medium", "high"],
|
||||
supportsThinking: true,
|
||||
supportsTemperature: true,
|
||||
maxOutputTokens: 128_000,
|
||||
toolCall: true,
|
||||
})
|
||||
expect(result.diagnostics).toMatchObject({
|
||||
resolutionMode: "snapshot-backed",
|
||||
canonicalization: { source: "canonical" },
|
||||
snapshot: { source: "bundled-snapshot" },
|
||||
variants: { source: "runtime" },
|
||||
})
|
||||
})
|
||||
|
||||
test("reads structured runtime capabilities from the SDK v2 shape", () => {
|
||||
const result = getModelCapabilities({
|
||||
providerID: "openai",
|
||||
modelID: "gpt-5.4",
|
||||
runtimeModel: {
|
||||
capabilities: {
|
||||
reasoning: true,
|
||||
temperature: false,
|
||||
toolcall: true,
|
||||
input: {
|
||||
text: true,
|
||||
image: true,
|
||||
},
|
||||
output: {
|
||||
text: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
bundledSnapshot,
|
||||
})
|
||||
|
||||
expect(result).toMatchObject({
|
||||
canonicalModelID: "gpt-5.4",
|
||||
reasoning: true,
|
||||
supportsThinking: true,
|
||||
supportsTemperature: false,
|
||||
toolCall: true,
|
||||
modalities: {
|
||||
input: ["text", "image"],
|
||||
output: ["text"],
|
||||
},
|
||||
})
|
||||
expect(result.diagnostics).toMatchObject({
|
||||
resolutionMode: "snapshot-backed",
|
||||
reasoning: { source: "runtime" },
|
||||
supportsThinking: { source: "runtime" },
|
||||
toolCall: { source: "runtime" },
|
||||
})
|
||||
})
|
||||
|
||||
test("respects root-level thinking flags when providers do not nest them under capabilities", () => {
|
||||
const result = getModelCapabilities({
|
||||
providerID: "custom-proxy",
|
||||
modelID: "gpt-5.4",
|
||||
runtimeModel: {
|
||||
supportsThinking: true,
|
||||
},
|
||||
bundledSnapshot,
|
||||
})
|
||||
|
||||
expect(result).toMatchObject({
|
||||
canonicalModelID: "gpt-5.4",
|
||||
supportsThinking: true,
|
||||
})
|
||||
expect(result.diagnostics).toMatchObject({
|
||||
supportsThinking: { source: "runtime" },
|
||||
})
|
||||
})
|
||||
|
||||
test("accepts runtime variant arrays without corrupting them into numeric keys", () => {
|
||||
const result = getModelCapabilities({
|
||||
providerID: "openai",
|
||||
modelID: "gpt-5.4",
|
||||
runtimeModel: {
|
||||
variants: ["low", "medium", "high", "xhigh"],
|
||||
},
|
||||
bundledSnapshot,
|
||||
})
|
||||
|
||||
expect(result.variants).toEqual(["low", "medium", "high", "xhigh"])
|
||||
})
|
||||
|
||||
test("normalizes the legacy Claude Opus thinking alias before snapshot lookup", () => {
|
||||
const result = getModelCapabilities({
|
||||
providerID: "anthropic",
|
||||
modelID: "claude-opus-4-6-thinking",
|
||||
bundledSnapshot,
|
||||
})
|
||||
|
||||
expect(result).toMatchObject({
|
||||
canonicalModelID: "claude-opus-4-6",
|
||||
family: "claude-opus",
|
||||
supportsThinking: true,
|
||||
supportsTemperature: true,
|
||||
maxOutputTokens: 128_000,
|
||||
})
|
||||
expect(result.diagnostics).toMatchObject({
|
||||
resolutionMode: "alias-backed",
|
||||
canonicalization: {
|
||||
source: "exact-alias",
|
||||
ruleID: "claude-opus-4-6-thinking-legacy-alias",
|
||||
},
|
||||
snapshot: { source: "bundled-snapshot" },
|
||||
})
|
||||
})
|
||||
|
||||
test("maps local gemini aliases to canonical models.dev entries", () => {
|
||||
const result = getModelCapabilities({
|
||||
providerID: "google",
|
||||
modelID: "gemini-3.1-pro-high",
|
||||
bundledSnapshot,
|
||||
})
|
||||
|
||||
expect(result).toMatchObject({
|
||||
canonicalModelID: "gemini-3.1-pro",
|
||||
family: "gemini",
|
||||
supportsThinking: true,
|
||||
supportsTemperature: true,
|
||||
maxOutputTokens: 65_000,
|
||||
})
|
||||
expect(result.diagnostics).toMatchObject({
|
||||
resolutionMode: "alias-backed",
|
||||
canonicalization: {
|
||||
source: "exact-alias",
|
||||
ruleID: "gemini-3.1-pro-tier-alias",
|
||||
},
|
||||
snapshot: { source: "bundled-snapshot" },
|
||||
})
|
||||
})
|
||||
|
||||
test("prefers runtime models.dev cache over bundled snapshot", () => {
|
||||
const runtimeSnapshot: ModelCapabilitiesSnapshot = {
|
||||
...bundledSnapshot,
|
||||
models: {
|
||||
...bundledSnapshot.models,
|
||||
"gpt-5.4": {
|
||||
...bundledSnapshot.models["gpt-5.4"],
|
||||
limit: {
|
||||
context: 1_050_000,
|
||||
output: 64_000,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
const result = getModelCapabilities({
|
||||
providerID: "openai",
|
||||
modelID: "gpt-5.4",
|
||||
bundledSnapshot,
|
||||
runtimeSnapshot,
|
||||
})
|
||||
|
||||
expect(result).toMatchObject({
|
||||
canonicalModelID: "gpt-5.4",
|
||||
maxOutputTokens: 64_000,
|
||||
supportsTemperature: false,
|
||||
})
|
||||
expect(result.diagnostics).toMatchObject({
|
||||
snapshot: { source: "runtime-snapshot" },
|
||||
maxOutputTokens: { source: "runtime-snapshot" },
|
||||
supportsTemperature: { source: "runtime-snapshot" },
|
||||
})
|
||||
})
|
||||
|
||||
test("falls back to heuristic family rules when no snapshot entry exists", () => {
|
||||
const result = getModelCapabilities({
|
||||
providerID: "openai",
|
||||
modelID: "o3-mini",
|
||||
bundledSnapshot,
|
||||
})
|
||||
|
||||
expect(result).toMatchObject({
|
||||
canonicalModelID: "o3-mini",
|
||||
family: "openai-reasoning",
|
||||
variants: ["low", "medium", "high"],
|
||||
reasoningEfforts: ["none", "minimal", "low", "medium", "high"],
|
||||
})
|
||||
expect(result.diagnostics).toMatchObject({
|
||||
resolutionMode: "heuristic-backed",
|
||||
snapshot: { source: "none" },
|
||||
family: { source: "heuristic" },
|
||||
reasoningEfforts: { source: "heuristic" },
|
||||
})
|
||||
})
|
||||
|
||||
test("detects prefixed o-series model IDs through the heuristic fallback", () => {
|
||||
const result = getModelCapabilities({
|
||||
providerID: "azure-openai",
|
||||
modelID: "openai/o3-mini",
|
||||
bundledSnapshot,
|
||||
})
|
||||
|
||||
expect(result).toMatchObject({
|
||||
canonicalModelID: "openai/o3-mini",
|
||||
family: "openai-reasoning",
|
||||
variants: ["low", "medium", "high"],
|
||||
reasoningEfforts: ["none", "minimal", "low", "medium", "high"],
|
||||
})
|
||||
expect(result.diagnostics).toMatchObject({
|
||||
resolutionMode: "heuristic-backed",
|
||||
snapshot: { source: "none" },
|
||||
family: { source: "heuristic" },
|
||||
})
|
||||
})
|
||||
|
||||
test("keeps every built-in OmO requirement model snapshot-backed", () => {
|
||||
const bundledSnapshot = getBundledModelCapabilitiesSnapshot()
|
||||
const requirementModels = new Set<string>()
|
||||
|
||||
for (const requirement of Object.values(AGENT_MODEL_REQUIREMENTS)) {
|
||||
for (const entry of requirement.fallbackChain) requirementModels.add(entry.model)
|
||||
}
|
||||
|
||||
for (const requirement of Object.values(CATEGORY_MODEL_REQUIREMENTS)) {
|
||||
for (const entry of requirement.fallbackChain) requirementModels.add(entry.model)
|
||||
}
|
||||
|
||||
for (const modelID of requirementModels) {
|
||||
const result = getModelCapabilities({
|
||||
providerID: "test-provider",
|
||||
modelID,
|
||||
bundledSnapshot,
|
||||
})
|
||||
|
||||
expect(result.diagnostics.resolutionMode).toBe("snapshot-backed")
|
||||
expect(result.diagnostics.snapshot.source).toBe("bundled-snapshot")
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,462 @@
|
||||
import bundledModelCapabilitiesSnapshotJson from "../generated/model-capabilities.generated.json"
|
||||
import { findProviderModelMetadata, type ModelMetadata } from "./connected-providers-cache"
|
||||
import { resolveModelIDAlias } from "./model-capability-aliases"
|
||||
import { detectHeuristicModelFamily } from "./model-capability-heuristics"
|
||||
|
||||
export type ModelCapabilitiesSnapshotEntry = {
|
||||
id: string
|
||||
family?: string
|
||||
reasoning?: boolean
|
||||
temperature?: boolean
|
||||
toolCall?: boolean
|
||||
modalities?: {
|
||||
input?: string[]
|
||||
output?: string[]
|
||||
}
|
||||
limit?: {
|
||||
context?: number
|
||||
input?: number
|
||||
output?: number
|
||||
}
|
||||
}
|
||||
|
||||
export type ModelCapabilitiesSnapshot = {
|
||||
generatedAt: string
|
||||
sourceUrl: string
|
||||
models: Record<string, ModelCapabilitiesSnapshotEntry>
|
||||
}
|
||||
|
||||
export type ModelCapabilities = {
|
||||
requestedModelID: string
|
||||
canonicalModelID: string
|
||||
family?: string
|
||||
variants?: string[]
|
||||
reasoningEfforts?: string[]
|
||||
reasoning?: boolean
|
||||
supportsThinking?: boolean
|
||||
supportsTemperature?: boolean
|
||||
supportsTopP?: boolean
|
||||
maxOutputTokens?: number
|
||||
toolCall?: boolean
|
||||
modalities?: {
|
||||
input?: string[]
|
||||
output?: string[]
|
||||
}
|
||||
diagnostics: ModelCapabilitiesDiagnostics
|
||||
}
|
||||
|
||||
type GetModelCapabilitiesInput = {
|
||||
providerID: string
|
||||
modelID: string
|
||||
runtimeModel?: ModelMetadata | Record<string, unknown>
|
||||
runtimeSnapshot?: ModelCapabilitiesSnapshot
|
||||
bundledSnapshot?: ModelCapabilitiesSnapshot
|
||||
}
|
||||
|
||||
type ModelCapabilityOverride = {
|
||||
variants?: string[]
|
||||
reasoningEfforts?: string[]
|
||||
supportsThinking?: boolean
|
||||
supportsTemperature?: boolean
|
||||
supportsTopP?: boolean
|
||||
}
|
||||
|
||||
type DiagnosticSource =
|
||||
| "none"
|
||||
| "runtime"
|
||||
| "runtime-snapshot"
|
||||
| "bundled-snapshot"
|
||||
| "override"
|
||||
| "heuristic"
|
||||
| "canonical"
|
||||
| "exact-alias"
|
||||
| "pattern-alias"
|
||||
|
||||
export type ModelCapabilitiesDiagnostics = {
|
||||
resolutionMode: "snapshot-backed" | "alias-backed" | "heuristic-backed" | "unknown"
|
||||
canonicalization: {
|
||||
source: "canonical" | "exact-alias" | "pattern-alias"
|
||||
ruleID?: string
|
||||
}
|
||||
snapshot: {
|
||||
source: "runtime-snapshot" | "bundled-snapshot" | "none"
|
||||
}
|
||||
family: { source: "snapshot" | "heuristic" | "none" }
|
||||
variants: { source: Exclude<DiagnosticSource, "runtime-snapshot" | "bundled-snapshot" | "exact-alias" | "pattern-alias"> }
|
||||
reasoningEfforts: { source: Exclude<DiagnosticSource, "runtime-snapshot" | "bundled-snapshot" | "canonical" | "exact-alias" | "pattern-alias" | "runtime"> }
|
||||
reasoning: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" }
|
||||
supportsThinking: { source: "runtime" | "override" | "heuristic" | "runtime-snapshot" | "bundled-snapshot" | "none" }
|
||||
supportsTemperature: { source: "runtime" | "override" | "runtime-snapshot" | "bundled-snapshot" | "none" }
|
||||
supportsTopP: { source: "runtime" | "override" | "none" }
|
||||
maxOutputTokens: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" }
|
||||
toolCall: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" }
|
||||
modalities: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" }
|
||||
}
|
||||
|
||||
const MODEL_ID_OVERRIDES: Record<string, ModelCapabilityOverride> = {}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
}
|
||||
|
||||
function normalizeLookupModelID(modelID: string): string {
|
||||
return modelID.trim().toLowerCase()
|
||||
}
|
||||
|
||||
function readBoolean(value: unknown): boolean | undefined {
|
||||
return typeof value === "boolean" ? value : undefined
|
||||
}
|
||||
|
||||
function readNumber(value: unknown): number | undefined {
|
||||
return typeof value === "number" ? value : undefined
|
||||
}
|
||||
|
||||
function readStringArray(value: unknown): string[] | undefined {
|
||||
if (!Array.isArray(value)) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const strings = value.filter((item): item is string => typeof item === "string")
|
||||
return strings.length > 0 ? strings : undefined
|
||||
}
|
||||
|
||||
function normalizeVariantKeys(value: unknown): string[] | undefined {
|
||||
const arrayVariants = readStringArray(value)
|
||||
if (arrayVariants) {
|
||||
return arrayVariants.map((variant) => variant.toLowerCase())
|
||||
}
|
||||
|
||||
if (!isRecord(value)) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const variants = Object.keys(value).map((variant) => variant.toLowerCase())
|
||||
return variants.length > 0 ? variants : undefined
|
||||
}
|
||||
|
||||
function readModalityKeys(value: unknown): string[] | undefined {
|
||||
const stringArray = readStringArray(value)
|
||||
if (stringArray) {
|
||||
return stringArray.map((entry) => entry.toLowerCase())
|
||||
}
|
||||
|
||||
if (!isRecord(value)) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const enabled = Object.entries(value)
|
||||
.filter(([, supported]) => supported === true)
|
||||
.map(([modality]) => modality.toLowerCase())
|
||||
|
||||
return enabled.length > 0 ? enabled : undefined
|
||||
}
|
||||
|
||||
function normalizeModalities(value: unknown): ModelCapabilities["modalities"] | undefined {
|
||||
if (!isRecord(value)) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const input = readModalityKeys(value.input)
|
||||
const output = readModalityKeys(value.output)
|
||||
|
||||
if (!input && !output) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return {
|
||||
...(input ? { input } : {}),
|
||||
...(output ? { output } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeSnapshot(snapshot: ModelCapabilitiesSnapshot | typeof bundledModelCapabilitiesSnapshotJson): ModelCapabilitiesSnapshot {
|
||||
return snapshot as ModelCapabilitiesSnapshot
|
||||
}
|
||||
|
||||
function getOverride(modelID: string): ModelCapabilityOverride | undefined {
|
||||
return MODEL_ID_OVERRIDES[normalizeLookupModelID(modelID)]
|
||||
}
|
||||
|
||||
function readRuntimeModelCapabilities(runtimeModel: Record<string, unknown> | undefined): Record<string, unknown> | undefined {
|
||||
return isRecord(runtimeModel?.capabilities) ? runtimeModel.capabilities : undefined
|
||||
}
|
||||
|
||||
function readRuntimeModelLimitOutput(runtimeModel: Record<string, unknown> | undefined): number | undefined {
|
||||
if (!runtimeModel) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const limit = isRecord(runtimeModel.limit)
|
||||
? runtimeModel.limit
|
||||
: readRuntimeModelCapabilities(runtimeModel)?.limit
|
||||
if (!isRecord(limit)) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return readNumber(limit.output)
|
||||
}
|
||||
|
||||
function readRuntimeModelBoolean(runtimeModel: Record<string, unknown> | undefined, keys: string[]): boolean | undefined {
|
||||
if (!runtimeModel) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const runtimeCapabilities = readRuntimeModelCapabilities(runtimeModel)
|
||||
|
||||
for (const key of keys) {
|
||||
const value = runtimeModel[key]
|
||||
if (typeof value === "boolean") {
|
||||
return value
|
||||
}
|
||||
|
||||
const capabilityValue = runtimeCapabilities?.[key]
|
||||
if (typeof capabilityValue === "boolean") {
|
||||
return capabilityValue
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
function readRuntimeModelModalities(runtimeModel: Record<string, unknown> | undefined): ModelCapabilities["modalities"] | undefined {
|
||||
if (!runtimeModel) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const rootModalities = normalizeModalities(runtimeModel.modalities)
|
||||
if (rootModalities) {
|
||||
return rootModalities
|
||||
}
|
||||
|
||||
const runtimeCapabilities = readRuntimeModelCapabilities(runtimeModel)
|
||||
if (!runtimeCapabilities) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const nestedModalities = normalizeModalities(runtimeCapabilities.modalities)
|
||||
if (nestedModalities) {
|
||||
return nestedModalities
|
||||
}
|
||||
|
||||
const capabilityModalities = normalizeModalities(runtimeCapabilities)
|
||||
if (capabilityModalities) {
|
||||
return capabilityModalities
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
function readRuntimeModelVariants(runtimeModel: Record<string, unknown> | undefined): string[] | undefined {
|
||||
if (!runtimeModel) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const rootVariants = normalizeVariantKeys(runtimeModel.variants)
|
||||
if (rootVariants) {
|
||||
return rootVariants
|
||||
}
|
||||
|
||||
const runtimeCapabilities = readRuntimeModelCapabilities(runtimeModel)
|
||||
if (!runtimeCapabilities) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return normalizeVariantKeys(runtimeCapabilities.variants)
|
||||
}
|
||||
|
||||
function readRuntimeModelTopPSupport(runtimeModel: Record<string, unknown> | undefined): boolean | undefined {
|
||||
return readRuntimeModelBoolean(runtimeModel, ["topP", "top_p"])
|
||||
}
|
||||
|
||||
function readRuntimeModelToolCallSupport(runtimeModel: Record<string, unknown> | undefined): boolean | undefined {
|
||||
return readRuntimeModelBoolean(runtimeModel, ["toolCall", "tool_call", "toolcall"])
|
||||
}
|
||||
|
||||
function readRuntimeModelReasoningSupport(runtimeModel: Record<string, unknown> | undefined): boolean | undefined {
|
||||
return readRuntimeModelBoolean(runtimeModel, ["reasoning"])
|
||||
}
|
||||
|
||||
function readRuntimeModelTemperatureSupport(runtimeModel: Record<string, unknown> | undefined): boolean | undefined {
|
||||
return readRuntimeModelBoolean(runtimeModel, ["temperature"])
|
||||
}
|
||||
|
||||
function readRuntimeModelThinkingSupport(runtimeModel: Record<string, unknown> | undefined): boolean | undefined {
|
||||
const capabilityValue = readRuntimeModelReasoningSupport(runtimeModel)
|
||||
if (capabilityValue !== undefined) {
|
||||
return capabilityValue
|
||||
}
|
||||
|
||||
const rootThinkingSupport = readRuntimeModelBoolean(runtimeModel, ["thinking", "supportsThinking"])
|
||||
if (rootThinkingSupport !== undefined) {
|
||||
return rootThinkingSupport
|
||||
}
|
||||
|
||||
const runtimeCapabilities = readRuntimeModelCapabilities(runtimeModel)
|
||||
if (!runtimeCapabilities) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
for (const key of ["thinking", "supportsThinking"] as const) {
|
||||
const value = runtimeCapabilities[key]
|
||||
if (typeof value === "boolean") {
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
function readRuntimeModel(runtimeModel: ModelMetadata | Record<string, unknown> | undefined): Record<string, unknown> | undefined {
|
||||
return isRecord(runtimeModel) ? runtimeModel : undefined
|
||||
}
|
||||
|
||||
const bundledModelCapabilitiesSnapshot = normalizeSnapshot(bundledModelCapabilitiesSnapshotJson)
|
||||
|
||||
export function getBundledModelCapabilitiesSnapshot(): ModelCapabilitiesSnapshot {
|
||||
return bundledModelCapabilitiesSnapshot
|
||||
}
|
||||
|
||||
export function getModelCapabilities(input: GetModelCapabilitiesInput): ModelCapabilities {
|
||||
const canonicalization = resolveModelIDAlias(input.modelID)
|
||||
const requestedModelID = canonicalization.requestedModelID
|
||||
const canonicalModelID = canonicalization.canonicalModelID
|
||||
const override = getOverride(input.modelID)
|
||||
const runtimeModel = readRuntimeModel(
|
||||
input.runtimeModel ?? findProviderModelMetadata(input.providerID, input.modelID),
|
||||
)
|
||||
const runtimeSnapshot = input.runtimeSnapshot
|
||||
const bundledSnapshot = input.bundledSnapshot ?? bundledModelCapabilitiesSnapshot
|
||||
const snapshotEntry = runtimeSnapshot?.models?.[canonicalModelID] ?? bundledSnapshot.models[canonicalModelID]
|
||||
const heuristicFamily = detectHeuristicModelFamily(canonicalModelID)
|
||||
const runtimeVariants = readRuntimeModelVariants(runtimeModel)
|
||||
const snapshotSource: ModelCapabilitiesDiagnostics["snapshot"]["source"] =
|
||||
runtimeSnapshot?.models?.[canonicalModelID]
|
||||
? "runtime-snapshot"
|
||||
: bundledSnapshot.models[canonicalModelID]
|
||||
? "bundled-snapshot"
|
||||
: "none"
|
||||
const familySource: ModelCapabilitiesDiagnostics["family"]["source"] =
|
||||
snapshotEntry?.family
|
||||
? "snapshot"
|
||||
: heuristicFamily?.family
|
||||
? "heuristic"
|
||||
: "none"
|
||||
const variantsSource: ModelCapabilitiesDiagnostics["variants"]["source"] =
|
||||
runtimeVariants
|
||||
? "runtime"
|
||||
: override?.variants
|
||||
? "override"
|
||||
: heuristicFamily?.variants
|
||||
? "heuristic"
|
||||
: "none"
|
||||
const reasoningEffortsSource: ModelCapabilitiesDiagnostics["reasoningEfforts"]["source"] =
|
||||
override?.reasoningEfforts
|
||||
? "override"
|
||||
: heuristicFamily?.reasoningEfforts
|
||||
? "heuristic"
|
||||
: "none"
|
||||
const reasoningSource: ModelCapabilitiesDiagnostics["reasoning"]["source"] =
|
||||
readRuntimeModelReasoningSupport(runtimeModel) !== undefined
|
||||
? "runtime"
|
||||
: snapshotEntry?.reasoning !== undefined
|
||||
? snapshotSource
|
||||
: "none"
|
||||
const supportsThinkingSource: ModelCapabilitiesDiagnostics["supportsThinking"]["source"] =
|
||||
override?.supportsThinking !== undefined
|
||||
? "override"
|
||||
: heuristicFamily?.supportsThinking !== undefined
|
||||
? "heuristic"
|
||||
: readRuntimeModelThinkingSupport(runtimeModel) !== undefined
|
||||
? "runtime"
|
||||
: snapshotEntry?.reasoning !== undefined
|
||||
? snapshotSource
|
||||
: "none"
|
||||
const supportsTemperatureSource: ModelCapabilitiesDiagnostics["supportsTemperature"]["source"] =
|
||||
readRuntimeModelTemperatureSupport(runtimeModel) !== undefined
|
||||
? "runtime"
|
||||
: override?.supportsTemperature !== undefined
|
||||
? "override"
|
||||
: snapshotEntry?.temperature !== undefined
|
||||
? snapshotSource
|
||||
: "none"
|
||||
const supportsTopPSource: ModelCapabilitiesDiagnostics["supportsTopP"]["source"] =
|
||||
readRuntimeModelTopPSupport(runtimeModel) !== undefined
|
||||
? "runtime"
|
||||
: override?.supportsTopP !== undefined
|
||||
? "override"
|
||||
: "none"
|
||||
const maxOutputTokensSource: ModelCapabilitiesDiagnostics["maxOutputTokens"]["source"] =
|
||||
readRuntimeModelLimitOutput(runtimeModel) !== undefined
|
||||
? "runtime"
|
||||
: snapshotEntry?.limit?.output !== undefined
|
||||
? snapshotSource
|
||||
: "none"
|
||||
const toolCallSource: ModelCapabilitiesDiagnostics["toolCall"]["source"] =
|
||||
readRuntimeModelToolCallSupport(runtimeModel) !== undefined
|
||||
? "runtime"
|
||||
: snapshotEntry?.toolCall !== undefined
|
||||
? snapshotSource
|
||||
: "none"
|
||||
const modalitiesSource: ModelCapabilitiesDiagnostics["modalities"]["source"] =
|
||||
readRuntimeModelModalities(runtimeModel) !== undefined
|
||||
? "runtime"
|
||||
: snapshotEntry?.modalities !== undefined
|
||||
? snapshotSource
|
||||
: "none"
|
||||
const resolutionMode: ModelCapabilitiesDiagnostics["resolutionMode"] =
|
||||
snapshotSource !== "none" && canonicalization.source === "canonical"
|
||||
? "snapshot-backed"
|
||||
: snapshotSource !== "none"
|
||||
? "alias-backed"
|
||||
: familySource === "heuristic" || variantsSource === "heuristic" || reasoningEffortsSource === "heuristic"
|
||||
? "heuristic-backed"
|
||||
: "unknown"
|
||||
|
||||
return {
|
||||
requestedModelID,
|
||||
canonicalModelID,
|
||||
family: snapshotEntry?.family ?? heuristicFamily?.family,
|
||||
variants: runtimeVariants ?? override?.variants ?? heuristicFamily?.variants,
|
||||
reasoningEfforts: override?.reasoningEfforts ?? heuristicFamily?.reasoningEfforts,
|
||||
reasoning: readRuntimeModelReasoningSupport(runtimeModel) ?? snapshotEntry?.reasoning,
|
||||
supportsThinking:
|
||||
override?.supportsThinking
|
||||
?? heuristicFamily?.supportsThinking
|
||||
?? readRuntimeModelThinkingSupport(runtimeModel)
|
||||
?? snapshotEntry?.reasoning,
|
||||
supportsTemperature:
|
||||
readRuntimeModelTemperatureSupport(runtimeModel)
|
||||
?? override?.supportsTemperature
|
||||
?? snapshotEntry?.temperature,
|
||||
supportsTopP:
|
||||
readRuntimeModelTopPSupport(runtimeModel)
|
||||
?? override?.supportsTopP,
|
||||
maxOutputTokens:
|
||||
readRuntimeModelLimitOutput(runtimeModel)
|
||||
?? snapshotEntry?.limit?.output,
|
||||
toolCall:
|
||||
readRuntimeModelToolCallSupport(runtimeModel)
|
||||
?? snapshotEntry?.toolCall,
|
||||
modalities:
|
||||
readRuntimeModelModalities(runtimeModel)
|
||||
?? snapshotEntry?.modalities,
|
||||
diagnostics: {
|
||||
resolutionMode,
|
||||
canonicalization: {
|
||||
source: canonicalization.source,
|
||||
...(canonicalization.ruleID ? { ruleID: canonicalization.ruleID } : {}),
|
||||
},
|
||||
snapshot: { source: snapshotSource },
|
||||
family: { source: familySource },
|
||||
variants: { source: variantsSource },
|
||||
reasoningEfforts: { source: reasoningEffortsSource },
|
||||
reasoning: { source: reasoningSource },
|
||||
supportsThinking: { source: supportsThinkingSource },
|
||||
supportsTemperature: { source: supportsTemperatureSource },
|
||||
supportsTopP: { source: supportsTopPSource },
|
||||
maxOutputTokens: { source: maxOutputTokensSource },
|
||||
toolCall: { source: toolCallSource },
|
||||
modalities: { source: modalitiesSource },
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
|
||||
import { resolveModelIDAlias } from "./model-capability-aliases"
|
||||
|
||||
describe("model-capability-aliases", () => {
|
||||
test("keeps canonical model IDs unchanged", () => {
|
||||
const result = resolveModelIDAlias("gpt-5.4")
|
||||
|
||||
expect(result).toEqual({
|
||||
requestedModelID: "gpt-5.4",
|
||||
canonicalModelID: "gpt-5.4",
|
||||
source: "canonical",
|
||||
})
|
||||
})
|
||||
|
||||
test("normalizes exact local tier aliases to canonical models.dev IDs", () => {
|
||||
const result = resolveModelIDAlias("gemini-3.1-pro-high")
|
||||
|
||||
expect(result).toEqual({
|
||||
requestedModelID: "gemini-3.1-pro-high",
|
||||
canonicalModelID: "gemini-3.1-pro",
|
||||
source: "exact-alias",
|
||||
ruleID: "gemini-3.1-pro-tier-alias",
|
||||
})
|
||||
})
|
||||
|
||||
test("does not resolve prototype keys as aliases", () => {
|
||||
const result = resolveModelIDAlias("constructor")
|
||||
|
||||
expect(result).toEqual({
|
||||
requestedModelID: "constructor",
|
||||
canonicalModelID: "constructor",
|
||||
source: "canonical",
|
||||
})
|
||||
})
|
||||
|
||||
test("normalizes legacy Claude thinking aliases through a named exact rule", () => {
|
||||
const result = resolveModelIDAlias("claude-opus-4-6-thinking")
|
||||
|
||||
expect(result).toEqual({
|
||||
requestedModelID: "claude-opus-4-6-thinking",
|
||||
canonicalModelID: "claude-opus-4-6",
|
||||
source: "exact-alias",
|
||||
ruleID: "claude-opus-4-6-thinking-legacy-alias",
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,103 @@
|
||||
export type ExactAliasRule = {
|
||||
aliasModelID: string
|
||||
ruleID: string
|
||||
canonicalModelID: string
|
||||
rationale: string
|
||||
}
|
||||
|
||||
export type PatternAliasRule = {
|
||||
ruleID: string
|
||||
description: string
|
||||
match: (normalizedModelID: string) => boolean
|
||||
canonicalize: (normalizedModelID: string) => string
|
||||
}
|
||||
|
||||
export type ModelIDAliasResolution = {
|
||||
requestedModelID: string
|
||||
canonicalModelID: string
|
||||
source: "canonical" | "exact-alias" | "pattern-alias"
|
||||
ruleID?: string
|
||||
}
|
||||
|
||||
const EXACT_ALIAS_RULES: ReadonlyArray<ExactAliasRule> = [
|
||||
{
|
||||
aliasModelID: "gemini-3.1-pro-high",
|
||||
ruleID: "gemini-3.1-pro-tier-alias",
|
||||
canonicalModelID: "gemini-3.1-pro",
|
||||
rationale: "OmO historically encoded Gemini tier selection in the model name instead of variant metadata.",
|
||||
},
|
||||
{
|
||||
aliasModelID: "gemini-3.1-pro-low",
|
||||
ruleID: "gemini-3.1-pro-tier-alias",
|
||||
canonicalModelID: "gemini-3.1-pro",
|
||||
rationale: "OmO historically encoded Gemini tier selection in the model name instead of variant metadata.",
|
||||
},
|
||||
{
|
||||
aliasModelID: "gemini-3-pro-high",
|
||||
ruleID: "gemini-3-pro-tier-alias",
|
||||
canonicalModelID: "gemini-3-pro-preview",
|
||||
rationale: "Legacy Gemini 3 tier suffixes still need to land on the canonical preview model.",
|
||||
},
|
||||
{
|
||||
aliasModelID: "gemini-3-pro-low",
|
||||
ruleID: "gemini-3-pro-tier-alias",
|
||||
canonicalModelID: "gemini-3-pro-preview",
|
||||
rationale: "Legacy Gemini 3 tier suffixes still need to land on the canonical preview model.",
|
||||
},
|
||||
{
|
||||
aliasModelID: "claude-opus-4-6-thinking",
|
||||
ruleID: "claude-opus-4-6-thinking-legacy-alias",
|
||||
canonicalModelID: "claude-opus-4-6",
|
||||
rationale: "OmO historically used a legacy compatibility suffix before models.dev shipped canonical thinking variants for newer Claude families.",
|
||||
},
|
||||
]
|
||||
|
||||
const EXACT_ALIAS_RULES_BY_MODEL: ReadonlyMap<string, ExactAliasRule> = new Map(
|
||||
EXACT_ALIAS_RULES.map((rule) => [rule.aliasModelID, rule]),
|
||||
)
|
||||
|
||||
const PATTERN_ALIAS_RULES: ReadonlyArray<PatternAliasRule> = []
|
||||
|
||||
function normalizeLookupModelID(modelID: string): string {
|
||||
return modelID.trim().toLowerCase()
|
||||
}
|
||||
|
||||
export function resolveModelIDAlias(modelID: string): ModelIDAliasResolution {
|
||||
const normalizedModelID = normalizeLookupModelID(modelID)
|
||||
const exactRule = EXACT_ALIAS_RULES_BY_MODEL.get(normalizedModelID)
|
||||
if (exactRule) {
|
||||
return {
|
||||
requestedModelID: normalizedModelID,
|
||||
canonicalModelID: exactRule.canonicalModelID,
|
||||
source: "exact-alias",
|
||||
ruleID: exactRule.ruleID,
|
||||
}
|
||||
}
|
||||
|
||||
for (const rule of PATTERN_ALIAS_RULES) {
|
||||
if (!rule.match(normalizedModelID)) {
|
||||
continue
|
||||
}
|
||||
|
||||
return {
|
||||
requestedModelID: normalizedModelID,
|
||||
canonicalModelID: rule.canonicalize(normalizedModelID),
|
||||
source: "pattern-alias",
|
||||
ruleID: rule.ruleID,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
requestedModelID: normalizedModelID,
|
||||
canonicalModelID: normalizedModelID,
|
||||
source: "canonical",
|
||||
}
|
||||
}
|
||||
|
||||
export function getExactModelIDAliasRules(): ReadonlyArray<ExactAliasRule> {
|
||||
return EXACT_ALIAS_RULES
|
||||
}
|
||||
|
||||
export function getPatternModelIDAliasRules(): ReadonlyArray<PatternAliasRule> {
|
||||
return PATTERN_ALIAS_RULES
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
|
||||
import type { ModelCapabilitiesSnapshot } from "./model-capabilities"
|
||||
import { getBundledModelCapabilitiesSnapshot } from "./model-capabilities"
|
||||
import {
|
||||
collectModelCapabilityGuardrailIssues,
|
||||
getBuiltInRequirementModelIDs,
|
||||
} from "./model-capability-guardrails"
|
||||
|
||||
describe("model-capability-guardrails", () => {
|
||||
test("keeps the current alias registry and built-in requirements aligned with the bundled snapshot", () => {
|
||||
const issues = collectModelCapabilityGuardrailIssues()
|
||||
|
||||
expect(issues).toEqual([])
|
||||
})
|
||||
|
||||
test("requires built-in requirement models to stay unique and sorted", () => {
|
||||
const modelIDs = getBuiltInRequirementModelIDs()
|
||||
|
||||
expect(modelIDs).toEqual([...modelIDs].sort())
|
||||
expect(new Set(modelIDs).size).toBe(modelIDs.length)
|
||||
expect(modelIDs).toContain("claude-opus-4-6")
|
||||
expect(modelIDs).toContain("gpt-5.4")
|
||||
expect(modelIDs).toContain("kimi-k2.5")
|
||||
})
|
||||
|
||||
test("flags exact aliases whose canonical target disappears from the snapshot", () => {
|
||||
const bundledSnapshot = getBundledModelCapabilitiesSnapshot()
|
||||
const brokenSnapshot: ModelCapabilitiesSnapshot = {
|
||||
...bundledSnapshot,
|
||||
models: Object.fromEntries(
|
||||
Object.entries(bundledSnapshot.models).filter(([modelID]) => modelID !== "gemini-3.1-pro"),
|
||||
),
|
||||
}
|
||||
|
||||
const issues = collectModelCapabilityGuardrailIssues({
|
||||
snapshot: brokenSnapshot,
|
||||
requirementModelIDs: [],
|
||||
})
|
||||
|
||||
expect(issues).toContainEqual(
|
||||
expect.objectContaining({
|
||||
kind: "alias-target-missing-from-snapshot",
|
||||
aliasModelID: "gemini-3.1-pro-high",
|
||||
canonicalModelID: "gemini-3.1-pro",
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
test("flags exact aliases when models.dev gains a canonical entry for the alias itself", () => {
|
||||
const bundledSnapshot = getBundledModelCapabilitiesSnapshot()
|
||||
const aliasCollisionSnapshot: ModelCapabilitiesSnapshot = {
|
||||
...bundledSnapshot,
|
||||
models: {
|
||||
...bundledSnapshot.models,
|
||||
"gemini-3.1-pro-high": {
|
||||
id: "gemini-3.1-pro-high",
|
||||
family: "gemini",
|
||||
reasoning: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
const issues = collectModelCapabilityGuardrailIssues({
|
||||
snapshot: aliasCollisionSnapshot,
|
||||
requirementModelIDs: [],
|
||||
})
|
||||
|
||||
expect(issues).toContainEqual(
|
||||
expect.objectContaining({
|
||||
kind: "exact-alias-collides-with-snapshot",
|
||||
aliasModelID: "gemini-3.1-pro-high",
|
||||
canonicalModelID: "gemini-3.1-pro",
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
test("flags built-in requirement models that rely on aliases instead of canonical IDs", () => {
|
||||
const issues = collectModelCapabilityGuardrailIssues({
|
||||
requirementModelIDs: ["gemini-3.1-pro-high"],
|
||||
})
|
||||
|
||||
expect(issues).toContainEqual(
|
||||
expect.objectContaining({
|
||||
kind: "built-in-model-relies-on-alias",
|
||||
modelID: "gemini-3.1-pro-high",
|
||||
canonicalModelID: "gemini-3.1-pro",
|
||||
ruleID: "gemini-3.1-pro-tier-alias",
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,149 @@
|
||||
import type { ModelCapabilitiesSnapshot } from "./model-capabilities"
|
||||
import { getBundledModelCapabilitiesSnapshot } from "./model-capabilities"
|
||||
import {
|
||||
getExactModelIDAliasRules,
|
||||
getPatternModelIDAliasRules,
|
||||
resolveModelIDAlias,
|
||||
} from "./model-capability-aliases"
|
||||
import { AGENT_MODEL_REQUIREMENTS, CATEGORY_MODEL_REQUIREMENTS } from "./model-requirements"
|
||||
|
||||
export type ModelCapabilityGuardrailIssue =
|
||||
| {
|
||||
kind: "alias-target-missing-from-snapshot"
|
||||
ruleID: string
|
||||
aliasModelID: string
|
||||
canonicalModelID: string
|
||||
message: string
|
||||
}
|
||||
| {
|
||||
kind: "exact-alias-collides-with-snapshot"
|
||||
ruleID: string
|
||||
aliasModelID: string
|
||||
canonicalModelID: string
|
||||
message: string
|
||||
}
|
||||
| {
|
||||
kind: "pattern-alias-collides-with-snapshot"
|
||||
ruleID: string
|
||||
modelID: string
|
||||
canonicalModelID: string
|
||||
message: string
|
||||
}
|
||||
| {
|
||||
kind: "built-in-model-relies-on-alias"
|
||||
modelID: string
|
||||
canonicalModelID: string
|
||||
ruleID: string
|
||||
message: string
|
||||
}
|
||||
| {
|
||||
kind: "built-in-model-missing-from-snapshot"
|
||||
modelID: string
|
||||
canonicalModelID: string
|
||||
message: string
|
||||
}
|
||||
|
||||
type CollectModelCapabilityGuardrailIssuesInput = {
|
||||
snapshot?: ModelCapabilitiesSnapshot
|
||||
requirementModelIDs?: Iterable<string>
|
||||
}
|
||||
|
||||
function normalizeLookupModelID(modelID: string): string {
|
||||
return modelID.trim().toLowerCase()
|
||||
}
|
||||
|
||||
export function getBuiltInRequirementModelIDs(): string[] {
|
||||
const modelIDs = new Set<string>()
|
||||
|
||||
for (const requirement of Object.values(AGENT_MODEL_REQUIREMENTS)) {
|
||||
for (const entry of requirement.fallbackChain) {
|
||||
modelIDs.add(entry.model)
|
||||
}
|
||||
}
|
||||
|
||||
for (const requirement of Object.values(CATEGORY_MODEL_REQUIREMENTS)) {
|
||||
for (const entry of requirement.fallbackChain) {
|
||||
modelIDs.add(entry.model)
|
||||
}
|
||||
}
|
||||
|
||||
return [...modelIDs].sort()
|
||||
}
|
||||
|
||||
export function collectModelCapabilityGuardrailIssues(
|
||||
input: CollectModelCapabilityGuardrailIssuesInput = {},
|
||||
): ModelCapabilityGuardrailIssue[] {
|
||||
const snapshot = input.snapshot ?? getBundledModelCapabilitiesSnapshot()
|
||||
const snapshotModelIDs = new Set(
|
||||
Object.keys(snapshot.models).map((modelID) => normalizeLookupModelID(modelID)),
|
||||
)
|
||||
const requirementModelIDs = input.requirementModelIDs ?? getBuiltInRequirementModelIDs()
|
||||
const issues: ModelCapabilityGuardrailIssue[] = []
|
||||
|
||||
for (const rule of getExactModelIDAliasRules()) {
|
||||
if (!snapshotModelIDs.has(rule.canonicalModelID)) {
|
||||
issues.push({
|
||||
kind: "alias-target-missing-from-snapshot",
|
||||
ruleID: rule.ruleID,
|
||||
aliasModelID: rule.aliasModelID,
|
||||
canonicalModelID: rule.canonicalModelID,
|
||||
message: `Alias ${rule.aliasModelID} points to missing snapshot model ${rule.canonicalModelID}.`,
|
||||
})
|
||||
}
|
||||
|
||||
if (snapshotModelIDs.has(rule.aliasModelID)) {
|
||||
issues.push({
|
||||
kind: "exact-alias-collides-with-snapshot",
|
||||
ruleID: rule.ruleID,
|
||||
aliasModelID: rule.aliasModelID,
|
||||
canonicalModelID: rule.canonicalModelID,
|
||||
message: `Alias ${rule.aliasModelID} now exists in models.dev and should be reviewed instead of force-mapping to ${rule.canonicalModelID}.`,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for (const rule of getPatternModelIDAliasRules()) {
|
||||
for (const modelID of snapshotModelIDs) {
|
||||
if (!rule.match(modelID)) {
|
||||
continue
|
||||
}
|
||||
|
||||
const canonicalModelID = rule.canonicalize(modelID)
|
||||
if (canonicalModelID === modelID) {
|
||||
continue
|
||||
}
|
||||
|
||||
issues.push({
|
||||
kind: "pattern-alias-collides-with-snapshot",
|
||||
ruleID: rule.ruleID,
|
||||
modelID,
|
||||
canonicalModelID,
|
||||
message: `Pattern alias ${rule.ruleID} would rewrite canonical snapshot model ${modelID} to ${canonicalModelID}.`,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for (const modelID of requirementModelIDs) {
|
||||
const aliasResolution = resolveModelIDAlias(modelID)
|
||||
if (aliasResolution.source !== "canonical") {
|
||||
issues.push({
|
||||
kind: "built-in-model-relies-on-alias",
|
||||
modelID: aliasResolution.requestedModelID,
|
||||
canonicalModelID: aliasResolution.canonicalModelID,
|
||||
ruleID: aliasResolution.ruleID ?? "unknown-alias-rule",
|
||||
message: `Built-in requirement model ${aliasResolution.requestedModelID} should be canonical and not rely on alias rule ${aliasResolution.ruleID}.`,
|
||||
})
|
||||
}
|
||||
|
||||
if (!snapshotModelIDs.has(aliasResolution.canonicalModelID)) {
|
||||
issues.push({
|
||||
kind: "built-in-model-missing-from-snapshot",
|
||||
modelID: aliasResolution.requestedModelID,
|
||||
canonicalModelID: aliasResolution.canonicalModelID,
|
||||
message: `Built-in requirement model ${aliasResolution.requestedModelID} resolves to ${aliasResolution.canonicalModelID}, which is missing from the bundled snapshot.`,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return issues
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { normalizeModelID } from "./model-normalization"
|
||||
|
||||
export type HeuristicModelFamilyDefinition = {
|
||||
family: string
|
||||
includes?: string[]
|
||||
pattern?: RegExp
|
||||
variants?: string[]
|
||||
reasoningEfforts?: string[]
|
||||
supportsThinking?: boolean
|
||||
}
|
||||
|
||||
export const HEURISTIC_MODEL_FAMILY_REGISTRY: ReadonlyArray<HeuristicModelFamilyDefinition> = [
|
||||
{
|
||||
family: "claude-opus",
|
||||
pattern: /claude(?:-\d+(?:-\d+)*)?-opus/,
|
||||
variants: ["low", "medium", "high", "max"],
|
||||
supportsThinking: true,
|
||||
},
|
||||
{
|
||||
family: "claude-non-opus",
|
||||
includes: ["claude"],
|
||||
variants: ["low", "medium", "high"],
|
||||
supportsThinking: true,
|
||||
},
|
||||
{
|
||||
family: "openai-reasoning",
|
||||
pattern: /(?:^|\/)o\d(?:$|-)/,
|
||||
variants: ["low", "medium", "high"],
|
||||
reasoningEfforts: ["none", "minimal", "low", "medium", "high"],
|
||||
},
|
||||
{
|
||||
family: "gpt-5",
|
||||
includes: ["gpt-5"],
|
||||
variants: ["low", "medium", "high", "xhigh"],
|
||||
reasoningEfforts: ["none", "minimal", "low", "medium", "high", "xhigh"],
|
||||
},
|
||||
{
|
||||
family: "gpt-legacy",
|
||||
includes: ["gpt"],
|
||||
variants: ["low", "medium", "high"],
|
||||
},
|
||||
{
|
||||
family: "gemini",
|
||||
includes: ["gemini"],
|
||||
variants: ["low", "medium", "high"],
|
||||
},
|
||||
{
|
||||
family: "kimi",
|
||||
includes: ["kimi", "k2"],
|
||||
variants: ["low", "medium", "high"],
|
||||
},
|
||||
{
|
||||
family: "glm",
|
||||
includes: ["glm"],
|
||||
variants: ["low", "medium", "high"],
|
||||
},
|
||||
{
|
||||
family: "minimax",
|
||||
includes: ["minimax"],
|
||||
variants: ["low", "medium", "high"],
|
||||
},
|
||||
{
|
||||
family: "deepseek",
|
||||
includes: ["deepseek"],
|
||||
variants: ["low", "medium", "high"],
|
||||
},
|
||||
{
|
||||
family: "mistral",
|
||||
includes: ["mistral", "codestral"],
|
||||
variants: ["low", "medium", "high"],
|
||||
},
|
||||
{
|
||||
family: "llama",
|
||||
includes: ["llama"],
|
||||
variants: ["low", "medium", "high"],
|
||||
},
|
||||
]
|
||||
|
||||
export function detectHeuristicModelFamily(modelID: string): HeuristicModelFamilyDefinition | undefined {
|
||||
const normalizedModelID = normalizeModelID(modelID).toLowerCase()
|
||||
|
||||
for (const definition of HEURISTIC_MODEL_FAMILY_REGISTRY) {
|
||||
if (definition.pattern?.test(normalizedModelID)) {
|
||||
return definition
|
||||
}
|
||||
|
||||
if (definition.includes?.some((value) => normalizedModelID.includes(value))) {
|
||||
return definition
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
@@ -2,6 +2,11 @@ export type FallbackEntry = {
|
||||
providers: string[];
|
||||
model: string;
|
||||
variant?: string; // Entry-specific variant (e.g., GPT→high, Opus→max)
|
||||
reasoningEffort?: string;
|
||||
temperature?: number;
|
||||
top_p?: number;
|
||||
maxTokens?: number;
|
||||
thinking?: { type: "enabled" | "disabled"; budgetTokens?: number };
|
||||
};
|
||||
|
||||
export type ModelRequirement = {
|
||||
|
||||
@@ -1,5 +1,16 @@
|
||||
import type { FallbackEntry } from "./model-requirements"
|
||||
|
||||
export interface DelegatedModelConfig {
|
||||
providerID: string
|
||||
modelID: string
|
||||
variant?: string
|
||||
reasoningEffort?: string
|
||||
temperature?: number
|
||||
top_p?: number
|
||||
maxTokens?: number
|
||||
thinking?: { type: "enabled" | "disabled"; budgetTokens?: number }
|
||||
}
|
||||
|
||||
export type ModelResolutionRequest = {
|
||||
intent?: {
|
||||
uiSelectedModel?: string
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import type { FallbackEntry } from "./model-requirements"
|
||||
import type { FallbackModelObject } from "../config/schema/fallback-models"
|
||||
import { normalizeModel } from "./model-normalization"
|
||||
import { resolveModelPipeline } from "./model-resolution-pipeline"
|
||||
import { KNOWN_VARIANTS } from "./known-variants"
|
||||
|
||||
export type ModelResolutionInput = {
|
||||
userModel?: string
|
||||
@@ -61,11 +63,45 @@ export function resolveModelWithFallback(
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes fallback_models config (which can be string or string[]) to string[]
|
||||
* Centralized helper to avoid duplicated normalization logic
|
||||
* Normalizes fallback_models config to a mixed array.
|
||||
* Accepts string, string[], or mixed arrays of strings and FallbackModelObject entries.
|
||||
*/
|
||||
export function normalizeFallbackModels(models: string | string[] | undefined): string[] | undefined {
|
||||
export function normalizeFallbackModels(
|
||||
models: string | (string | FallbackModelObject)[] | undefined,
|
||||
): (string | FallbackModelObject)[] | undefined {
|
||||
if (!models) return undefined
|
||||
if (typeof models === "string") return [models]
|
||||
return models
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts plain model strings from a mixed fallback models array.
|
||||
* Object entries are flattened to "model" or "model(variant)" strings.
|
||||
* Use this when consumers need string[] (e.g., resolveModelForDelegateTask).
|
||||
*/
|
||||
export function flattenToFallbackModelStrings(
|
||||
models: (string | FallbackModelObject)[] | undefined,
|
||||
): string[] | undefined {
|
||||
if (!models) return undefined
|
||||
return models.map((entry) => {
|
||||
if (typeof entry === "string") return entry
|
||||
const variant = entry.variant
|
||||
if (variant) {
|
||||
// Strip any supported inline variant syntax before appending explicit override.
|
||||
// Supports both parenthesized and space-suffix forms so we don't emit
|
||||
// invalid strings like "provider/model high(low)".
|
||||
const model = entry.model
|
||||
.replace(/\([^()]+\)\s*$/, "")
|
||||
.replace(/\s+([a-z][a-z0-9_-]*)\s*$/i, (match, suffix) => {
|
||||
const normalized = String(suffix).toLowerCase()
|
||||
return KNOWN_VARIANTS.has(normalized)
|
||||
? ""
|
||||
: match
|
||||
})
|
||||
.trim()
|
||||
return `${model}(${variant})`
|
||||
}
|
||||
// No explicit variant — preserve model string as-is (including any inline variant)
|
||||
return entry.model
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,513 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
|
||||
import { resolveCompatibleModelSettings } from "./model-settings-compatibility"
|
||||
|
||||
describe("resolveCompatibleModelSettings", () => {
|
||||
test("keeps supported Claude Opus variant unchanged", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "anthropic",
|
||||
modelID: "claude-opus-4-6",
|
||||
desired: { variant: "max" },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: "max",
|
||||
reasoningEffort: undefined,
|
||||
changes: [],
|
||||
})
|
||||
})
|
||||
|
||||
test("uses model metadata first for variant support", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "anthropic",
|
||||
modelID: "claude-opus-4-6",
|
||||
desired: { variant: "max" },
|
||||
capabilities: { variants: ["low", "medium", "high"] },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: "high",
|
||||
reasoningEffort: undefined,
|
||||
changes: [
|
||||
{
|
||||
field: "variant",
|
||||
from: "max",
|
||||
to: "high",
|
||||
reason: "unsupported-by-model-metadata",
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
test("prefers metadata over family heuristics even when family would allow a higher level", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "anthropic",
|
||||
modelID: "claude-opus-4-6",
|
||||
desired: { variant: "max" },
|
||||
capabilities: { variants: ["low", "medium"] },
|
||||
})
|
||||
|
||||
expect(result.variant).toBe("medium")
|
||||
expect(result.changes).toEqual([
|
||||
{
|
||||
field: "variant",
|
||||
from: "max",
|
||||
to: "medium",
|
||||
reason: "unsupported-by-model-metadata",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("downgrades unsupported Claude Sonnet max variant to high when metadata is absent", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "anthropic",
|
||||
modelID: "claude-sonnet-4-6",
|
||||
desired: { variant: "max" },
|
||||
})
|
||||
|
||||
expect(result.variant).toBe("high")
|
||||
expect(result.changes).toEqual([
|
||||
{
|
||||
field: "variant",
|
||||
from: "max",
|
||||
to: "high",
|
||||
reason: "unsupported-by-model-family",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("keeps supported GPT reasoningEffort unchanged", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "gpt-5.4",
|
||||
desired: { reasoningEffort: "high" },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: undefined,
|
||||
reasoningEffort: "high",
|
||||
changes: [],
|
||||
})
|
||||
})
|
||||
|
||||
test("keeps supported OpenAI reasoning-family effort for o-series models", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "o3-mini",
|
||||
desired: { reasoningEffort: "high" },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: undefined,
|
||||
reasoningEffort: "high",
|
||||
changes: [],
|
||||
})
|
||||
})
|
||||
|
||||
test("does not record case-only normalization as a compatibility downgrade", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "gpt-5.4",
|
||||
desired: { variant: "HIGH", reasoningEffort: "HIGH" },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: "high",
|
||||
reasoningEffort: "high",
|
||||
changes: [],
|
||||
})
|
||||
})
|
||||
|
||||
test("drops reasoningEffort for standard GPT models (gpt-4.1)", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "gpt-4.1",
|
||||
desired: { reasoningEffort: "high" },
|
||||
})
|
||||
|
||||
expect(result.reasoningEffort).toBeUndefined()
|
||||
expect(result.changes).toEqual([
|
||||
{
|
||||
field: "reasoningEffort",
|
||||
from: "high",
|
||||
to: undefined,
|
||||
reason: "unsupported-by-model-family",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("drops reasoningEffort for Claude family", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "anthropic",
|
||||
modelID: "claude-sonnet-4-6",
|
||||
desired: { reasoningEffort: "high" },
|
||||
})
|
||||
|
||||
expect(result.reasoningEffort).toBeUndefined()
|
||||
expect(result.changes).toEqual([
|
||||
{
|
||||
field: "reasoningEffort",
|
||||
from: "high",
|
||||
to: undefined,
|
||||
reason: "unsupported-by-model-family",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("handles combined variant and reasoningEffort normalization", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "anthropic",
|
||||
modelID: "claude-sonnet-4-6",
|
||||
desired: { variant: "max", reasoningEffort: "high" },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: "high",
|
||||
reasoningEffort: undefined,
|
||||
changes: [
|
||||
{
|
||||
field: "variant",
|
||||
from: "max",
|
||||
to: "high",
|
||||
reason: "unsupported-by-model-family",
|
||||
},
|
||||
{
|
||||
field: "reasoningEffort",
|
||||
from: "high",
|
||||
to: undefined,
|
||||
reason: "unsupported-by-model-family",
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
test("treats unknown model families conservatively by dropping unsupported settings", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "mystery",
|
||||
modelID: "mystery-model-1",
|
||||
desired: { variant: "max", reasoningEffort: "high" },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: undefined,
|
||||
reasoningEffort: undefined,
|
||||
changes: [
|
||||
{
|
||||
field: "variant",
|
||||
from: "max",
|
||||
to: undefined,
|
||||
reason: "unknown-model-family",
|
||||
},
|
||||
{
|
||||
field: "reasoningEffort",
|
||||
from: "high",
|
||||
to: undefined,
|
||||
reason: "unknown-model-family",
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
// Provider-agnostic detection: model ID is the source of truth, not provider ID
|
||||
test("detects Claude via any provider (provider-agnostic)", () => {
|
||||
for (const providerID of ["anthropic", "aws-bedrock", "bedrock", "amazon-bedrock", "opencode", "my-custom-proxy", "google-vertex-anthropic"]) {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID,
|
||||
modelID: "claude-sonnet-4-6",
|
||||
desired: { variant: "max" },
|
||||
})
|
||||
|
||||
expect(result.variant).toBe("high")
|
||||
expect(result.changes[0]?.reason).toBe("unsupported-by-model-family")
|
||||
}
|
||||
})
|
||||
|
||||
test("detects Claude 3 Opus via any provider", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "some-unknown-proxy",
|
||||
modelID: "claude-3-opus-20240229",
|
||||
desired: { variant: "max" },
|
||||
})
|
||||
|
||||
expect(result.variant).toBe("max")
|
||||
expect(result.changes).toEqual([])
|
||||
})
|
||||
|
||||
test("detects OpenAI reasoning models without requiring openai provider", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "azure-openai",
|
||||
modelID: "o3-mini",
|
||||
desired: { reasoningEffort: "high" },
|
||||
})
|
||||
|
||||
expect(result.reasoningEffort).toBe("high")
|
||||
expect(result.changes).toEqual([])
|
||||
})
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Registry coverage — every model family from FAMILY_CAPABILITIES
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe("model family registry coverage", () => {
|
||||
const familyCases: Array<{
|
||||
name: string
|
||||
modelID: string
|
||||
expectedVariants: string[]
|
||||
hasReasoningEffort: boolean
|
||||
}> = [
|
||||
{ name: "Gemini", modelID: "gemini-3.1-pro", expectedVariants: ["low", "medium", "high"], hasReasoningEffort: false },
|
||||
{ name: "Kimi (kimi)", modelID: "kimi-k2.5", expectedVariants: ["low", "medium", "high"], hasReasoningEffort: false },
|
||||
{ name: "Kimi (k2)", modelID: "k2-v2", expectedVariants: ["low", "medium", "high"], hasReasoningEffort: false },
|
||||
{ name: "GLM", modelID: "glm-5", expectedVariants: ["low", "medium", "high"], hasReasoningEffort: false },
|
||||
{ name: "Minimax", modelID: "minimax-m2.5", expectedVariants: ["low", "medium", "high"], hasReasoningEffort: false },
|
||||
{ name: "DeepSeek", modelID: "deepseek-r2", expectedVariants: ["low", "medium", "high"], hasReasoningEffort: false },
|
||||
{ name: "Mistral", modelID: "mistral-large-next", expectedVariants: ["low", "medium", "high"], hasReasoningEffort: false },
|
||||
{ name: "Codestral → Mistral", modelID: "codestral-2506", expectedVariants: ["low", "medium", "high"], hasReasoningEffort: false },
|
||||
{ name: "Llama", modelID: "llama-4-maverick", expectedVariants: ["low", "medium", "high"], hasReasoningEffort: false },
|
||||
]
|
||||
|
||||
for (const { name, modelID, expectedVariants, hasReasoningEffort } of familyCases) {
|
||||
test(`${name} (${modelID}): keeps supported variant`, () => {
|
||||
const highest = expectedVariants[expectedVariants.length - 1]
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "any-provider",
|
||||
modelID,
|
||||
desired: { variant: highest },
|
||||
})
|
||||
|
||||
expect(result.variant).toBe(highest)
|
||||
expect(result.changes).toEqual([])
|
||||
})
|
||||
|
||||
test(`${name} (${modelID}): downgrades unsupported variant`, () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "any-provider",
|
||||
modelID,
|
||||
desired: { variant: "max" },
|
||||
})
|
||||
|
||||
const highest = expectedVariants[expectedVariants.length - 1]
|
||||
expect(result.variant).toBe(highest)
|
||||
expect(result.changes[0]?.reason).toBe("unsupported-by-model-family")
|
||||
})
|
||||
|
||||
test(`${name} (${modelID}): ${hasReasoningEffort ? "keeps" : "drops"} reasoningEffort`, () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "any-provider",
|
||||
modelID,
|
||||
desired: { reasoningEffort: "high" },
|
||||
})
|
||||
|
||||
if (hasReasoningEffort) {
|
||||
expect(result.reasoningEffort).toBe("high")
|
||||
expect(result.changes).toEqual([])
|
||||
} else {
|
||||
expect(result.reasoningEffort).toBeUndefined()
|
||||
expect(result.changes[0]?.reason).toBe("unsupported-by-model-family")
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
// GPT-5 specific: supports xhigh variant and xhigh reasoningEffort
|
||||
test("GPT-5 keeps xhigh variant and reasoningEffort", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "gpt-5.4",
|
||||
desired: { variant: "xhigh", reasoningEffort: "xhigh" },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: "xhigh",
|
||||
reasoningEffort: "xhigh",
|
||||
changes: [],
|
||||
})
|
||||
})
|
||||
|
||||
test("GPT-5 downgrades unsupported max variant to xhigh", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "gpt-5.4",
|
||||
desired: { variant: "max" },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: "xhigh",
|
||||
reasoningEffort: undefined,
|
||||
changes: [
|
||||
{
|
||||
field: "variant",
|
||||
from: "max",
|
||||
to: "xhigh",
|
||||
reason: "unsupported-by-model-family",
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
// Reasoning effort: "none" and "minimal" are valid per Vercel AI SDK
|
||||
test("GPT-5 keeps none reasoningEffort", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "gpt-5.4",
|
||||
desired: { reasoningEffort: "none" },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: undefined,
|
||||
reasoningEffort: "none",
|
||||
changes: [],
|
||||
})
|
||||
})
|
||||
|
||||
test("GPT-5 keeps minimal reasoningEffort", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "gpt-5.4",
|
||||
desired: { reasoningEffort: "minimal" },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: undefined,
|
||||
reasoningEffort: "minimal",
|
||||
changes: [],
|
||||
})
|
||||
})
|
||||
|
||||
test("o-series keeps none reasoningEffort", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "o3-mini",
|
||||
desired: { reasoningEffort: "none" },
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: undefined,
|
||||
reasoningEffort: "none",
|
||||
changes: [],
|
||||
})
|
||||
})
|
||||
|
||||
// Reasoning effort downgrade within families that support it
|
||||
test("o-series downgrades xhigh reasoningEffort to high", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "o3-mini",
|
||||
desired: { reasoningEffort: "xhigh" },
|
||||
})
|
||||
|
||||
expect(result.reasoningEffort).toBe("high")
|
||||
expect(result.changes).toEqual([
|
||||
{
|
||||
field: "reasoningEffort",
|
||||
from: "xhigh",
|
||||
to: "high",
|
||||
reason: "unsupported-by-model-family",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("GPT-5 keeps xhigh but would downgrade a hypothetical beyond-max level", () => {
|
||||
// GPT-5 supports up to "xhigh" — verify the ladder works by requesting
|
||||
// a value that IS in the ladder but NOT in the family's allowed list.
|
||||
// Since "xhigh" is the max for GPT-5 reasoningEffort, we verify it stays.
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "gpt-5.4",
|
||||
desired: { reasoningEffort: "xhigh" },
|
||||
})
|
||||
|
||||
expect(result.reasoningEffort).toBe("xhigh")
|
||||
expect(result.changes).toEqual([])
|
||||
})
|
||||
|
||||
test("o-series downgrades unsupported variant to high", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "o3-mini",
|
||||
desired: { variant: "max" },
|
||||
})
|
||||
|
||||
expect(result.variant).toBe("high")
|
||||
expect(result.changes).toEqual([
|
||||
{
|
||||
field: "variant",
|
||||
from: "max",
|
||||
to: "high",
|
||||
reason: "unsupported-by-model-family",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("drops unsupported temperature when capability metadata disables it", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "gpt-5.4",
|
||||
desired: { temperature: 0.7 },
|
||||
capabilities: { supportsTemperature: false },
|
||||
})
|
||||
|
||||
expect(result.temperature).toBeUndefined()
|
||||
expect(result.changes).toEqual([
|
||||
{
|
||||
field: "temperature",
|
||||
from: "0.7",
|
||||
to: undefined,
|
||||
reason: "unsupported-by-model-metadata",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("drops thinking when model capabilities say it is unsupported", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "gpt-5.4",
|
||||
desired: { thinking: { type: "enabled", budgetTokens: 4096 } },
|
||||
capabilities: { supportsThinking: false },
|
||||
})
|
||||
|
||||
expect(result.thinking).toBeUndefined()
|
||||
expect(result.changes).toEqual([
|
||||
{
|
||||
field: "thinking",
|
||||
from: "{\"type\":\"enabled\",\"budgetTokens\":4096}",
|
||||
to: undefined,
|
||||
reason: "unsupported-by-model-metadata",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test("clamps maxTokens to the model output limit", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "openai",
|
||||
modelID: "gpt-5.4",
|
||||
desired: { maxTokens: 200_000 },
|
||||
capabilities: { maxOutputTokens: 128_000 },
|
||||
})
|
||||
|
||||
expect(result.maxTokens).toBe(128_000)
|
||||
expect(result.changes).toEqual([
|
||||
{
|
||||
field: "maxTokens",
|
||||
from: "200000",
|
||||
to: "128000",
|
||||
reason: "max-output-limit",
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
// Passthrough: undefined desired values produce no changes
|
||||
test("no-op when desired settings are empty", () => {
|
||||
const result = resolveCompatibleModelSettings({
|
||||
providerID: "anthropic",
|
||||
modelID: "claude-opus-4-6",
|
||||
desired: {},
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
variant: undefined,
|
||||
reasoningEffort: undefined,
|
||||
changes: [],
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,211 @@
|
||||
import { detectHeuristicModelFamily } from "./model-capability-heuristics"
|
||||
|
||||
type CompatibilityField = "variant" | "reasoningEffort" | "temperature" | "topP" | "maxTokens" | "thinking"
|
||||
|
||||
type DesiredModelSettings = {
|
||||
variant?: string
|
||||
reasoningEffort?: string
|
||||
temperature?: number
|
||||
topP?: number
|
||||
maxTokens?: number
|
||||
thinking?: Record<string, unknown>
|
||||
}
|
||||
|
||||
type CompatibilityCapabilities = {
|
||||
variants?: string[]
|
||||
reasoningEfforts?: string[]
|
||||
supportsTemperature?: boolean
|
||||
supportsTopP?: boolean
|
||||
maxOutputTokens?: number
|
||||
supportsThinking?: boolean
|
||||
}
|
||||
|
||||
export type ModelSettingsCompatibilityInput = {
|
||||
providerID: string
|
||||
modelID: string
|
||||
desired: DesiredModelSettings
|
||||
capabilities?: CompatibilityCapabilities
|
||||
}
|
||||
|
||||
export type ModelSettingsCompatibilityChange = {
|
||||
field: CompatibilityField
|
||||
from: string
|
||||
to?: string
|
||||
reason:
|
||||
| "unsupported-by-model-family"
|
||||
| "unknown-model-family"
|
||||
| "unsupported-by-model-metadata"
|
||||
| "max-output-limit"
|
||||
}
|
||||
|
||||
export type ModelSettingsCompatibilityResult = {
|
||||
variant?: string
|
||||
reasoningEffort?: string
|
||||
temperature?: number
|
||||
topP?: number
|
||||
maxTokens?: number
|
||||
thinking?: Record<string, unknown>
|
||||
changes: ModelSettingsCompatibilityChange[]
|
||||
}
|
||||
|
||||
const VARIANT_LADDER = ["low", "medium", "high", "xhigh", "max"]
|
||||
const REASONING_LADDER = ["none", "minimal", "low", "medium", "high", "xhigh"]
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Generic resolution — one function for both fields
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function downgradeWithinLadder(value: string, allowed: string[], ladder: string[]): string | undefined {
|
||||
const requestedIndex = ladder.indexOf(value)
|
||||
if (requestedIndex === -1) return undefined
|
||||
|
||||
for (let index = requestedIndex; index >= 0; index -= 1) {
|
||||
if (allowed.includes(ladder[index])) {
|
||||
return ladder[index]
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
function normalizeCapabilitiesVariants(capabilities: CompatibilityCapabilities | undefined): string[] | undefined {
|
||||
if (!capabilities?.variants || capabilities.variants.length === 0) {
|
||||
return undefined
|
||||
}
|
||||
return capabilities.variants.map((v) => v.toLowerCase())
|
||||
}
|
||||
|
||||
function normalizeCapabilitiesReasoningEfforts(capabilities: CompatibilityCapabilities | undefined): string[] | undefined {
|
||||
if (!capabilities?.reasoningEfforts || capabilities.reasoningEfforts.length === 0) {
|
||||
return undefined
|
||||
}
|
||||
return capabilities.reasoningEfforts.map((value) => value.toLowerCase())
|
||||
}
|
||||
|
||||
type FieldResolution = { value?: string; reason?: ModelSettingsCompatibilityChange["reason"] }
|
||||
|
||||
function resolveField(
|
||||
normalized: string,
|
||||
familyCaps: string[] | undefined,
|
||||
ladder: string[],
|
||||
familyKnown: boolean,
|
||||
metadataOverride?: string[],
|
||||
): FieldResolution {
|
||||
// Priority 1: runtime metadata from provider
|
||||
if (metadataOverride) {
|
||||
if (metadataOverride.includes(normalized)) return { value: normalized }
|
||||
return {
|
||||
value: downgradeWithinLadder(normalized, metadataOverride, ladder),
|
||||
reason: "unsupported-by-model-metadata",
|
||||
}
|
||||
}
|
||||
|
||||
// Priority 2: family heuristic from registry
|
||||
if (familyCaps) {
|
||||
if (familyCaps.includes(normalized)) return { value: normalized }
|
||||
return {
|
||||
value: downgradeWithinLadder(normalized, familyCaps, ladder),
|
||||
reason: "unsupported-by-model-family",
|
||||
}
|
||||
}
|
||||
|
||||
// Known family but field not in registry (e.g. Claude + reasoningEffort)
|
||||
if (familyKnown) {
|
||||
return { value: undefined, reason: "unsupported-by-model-family" }
|
||||
}
|
||||
|
||||
// Unknown family — drop the value
|
||||
return { value: undefined, reason: "unknown-model-family" }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function resolveCompatibleModelSettings(
|
||||
input: ModelSettingsCompatibilityInput,
|
||||
): ModelSettingsCompatibilityResult {
|
||||
const family = detectHeuristicModelFamily(input.modelID)
|
||||
const familyKnown = family !== undefined
|
||||
const changes: ModelSettingsCompatibilityChange[] = []
|
||||
const metadataVariants = normalizeCapabilitiesVariants(input.capabilities)
|
||||
const metadataReasoningEfforts = normalizeCapabilitiesReasoningEfforts(input.capabilities)
|
||||
|
||||
let variant = input.desired.variant
|
||||
if (variant !== undefined) {
|
||||
const normalized = variant.toLowerCase()
|
||||
const resolved = resolveField(normalized, family?.variants, VARIANT_LADDER, familyKnown, metadataVariants)
|
||||
if (resolved.value !== normalized && resolved.reason) {
|
||||
changes.push({ field: "variant", from: variant, to: resolved.value, reason: resolved.reason })
|
||||
}
|
||||
variant = resolved.value
|
||||
}
|
||||
|
||||
let reasoningEffort = input.desired.reasoningEffort
|
||||
if (reasoningEffort !== undefined) {
|
||||
const normalized = reasoningEffort.toLowerCase()
|
||||
const resolved = resolveField(normalized, family?.reasoningEfforts, REASONING_LADDER, familyKnown, metadataReasoningEfforts)
|
||||
if (resolved.value !== normalized && resolved.reason) {
|
||||
changes.push({ field: "reasoningEffort", from: reasoningEffort, to: resolved.value, reason: resolved.reason })
|
||||
}
|
||||
reasoningEffort = resolved.value
|
||||
}
|
||||
|
||||
let temperature = input.desired.temperature
|
||||
if (temperature !== undefined && input.capabilities?.supportsTemperature === false) {
|
||||
changes.push({
|
||||
field: "temperature",
|
||||
from: String(temperature),
|
||||
to: undefined,
|
||||
reason: "unsupported-by-model-metadata",
|
||||
})
|
||||
temperature = undefined
|
||||
}
|
||||
|
||||
let topP = input.desired.topP
|
||||
if (topP !== undefined && input.capabilities?.supportsTopP === false) {
|
||||
changes.push({
|
||||
field: "topP",
|
||||
from: String(topP),
|
||||
to: undefined,
|
||||
reason: "unsupported-by-model-metadata",
|
||||
})
|
||||
topP = undefined
|
||||
}
|
||||
|
||||
let maxTokens = input.desired.maxTokens
|
||||
if (
|
||||
maxTokens !== undefined &&
|
||||
input.capabilities?.maxOutputTokens !== undefined &&
|
||||
maxTokens > input.capabilities.maxOutputTokens
|
||||
) {
|
||||
changes.push({
|
||||
field: "maxTokens",
|
||||
from: String(maxTokens),
|
||||
to: String(input.capabilities.maxOutputTokens),
|
||||
reason: "max-output-limit",
|
||||
})
|
||||
maxTokens = input.capabilities.maxOutputTokens
|
||||
}
|
||||
|
||||
let thinking = input.desired.thinking
|
||||
if (thinking !== undefined && input.capabilities?.supportsThinking === false) {
|
||||
changes.push({
|
||||
field: "thinking",
|
||||
from: JSON.stringify(thinking),
|
||||
to: undefined,
|
||||
reason: "unsupported-by-model-metadata",
|
||||
})
|
||||
thinking = undefined
|
||||
}
|
||||
|
||||
return {
|
||||
variant,
|
||||
reasoningEffort,
|
||||
...(input.desired.temperature !== undefined ? { temperature } : {}),
|
||||
...(input.desired.topP !== undefined ? { topP } : {}),
|
||||
...(input.desired.maxTokens !== undefined ? { maxTokens } : {}),
|
||||
...(input.desired.thinking !== undefined ? { thinking } : {}),
|
||||
changes,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import { clearSessionPromptParams, setSessionPromptParams } from "./session-prompt-params-state"
|
||||
|
||||
type PromptParamModel = {
|
||||
temperature?: number
|
||||
top_p?: number
|
||||
reasoningEffort?: string
|
||||
maxTokens?: number
|
||||
thinking?: { type: "enabled" | "disabled"; budgetTokens?: number }
|
||||
}
|
||||
|
||||
export function applySessionPromptParams(
|
||||
sessionID: string,
|
||||
model: PromptParamModel | undefined,
|
||||
): void {
|
||||
if (!model) {
|
||||
clearSessionPromptParams(sessionID)
|
||||
return
|
||||
}
|
||||
|
||||
const promptOptions: Record<string, unknown> = {
|
||||
...(model.reasoningEffort ? { reasoningEffort: model.reasoningEffort } : {}),
|
||||
...(model.thinking ? { thinking: model.thinking } : {}),
|
||||
...(model.maxTokens !== undefined ? { maxTokens: model.maxTokens } : {}),
|
||||
}
|
||||
|
||||
setSessionPromptParams(sessionID, {
|
||||
...(model.temperature !== undefined ? { temperature: model.temperature } : {}),
|
||||
...(model.top_p !== undefined ? { topP: model.top_p } : {}),
|
||||
...(Object.keys(promptOptions).length > 0 ? { options: promptOptions } : {}),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import { afterEach, describe, expect, test } from "bun:test"
|
||||
|
||||
import {
|
||||
clearAllSessionPromptParams,
|
||||
clearSessionPromptParams,
|
||||
getSessionPromptParams,
|
||||
setSessionPromptParams,
|
||||
} from "./session-prompt-params-state"
|
||||
|
||||
describe("session-prompt-params-state", () => {
|
||||
afterEach(() => {
|
||||
clearAllSessionPromptParams()
|
||||
})
|
||||
|
||||
test("stores and returns prompt params by session", () => {
|
||||
//#given
|
||||
const sessionID = "ses_prompt_params"
|
||||
const params = {
|
||||
temperature: 0.4,
|
||||
topP: 0.7,
|
||||
options: {
|
||||
reasoningEffort: "high",
|
||||
maxTokens: 4096,
|
||||
},
|
||||
}
|
||||
|
||||
//#when
|
||||
setSessionPromptParams(sessionID, params)
|
||||
|
||||
//#then
|
||||
expect(getSessionPromptParams(sessionID)).toEqual(params)
|
||||
})
|
||||
|
||||
test("returns copies so callers cannot mutate stored state", () => {
|
||||
//#given
|
||||
const sessionID = "ses_prompt_params_copy"
|
||||
setSessionPromptParams(sessionID, {
|
||||
temperature: 0.2,
|
||||
options: { reasoningEffort: "medium" },
|
||||
})
|
||||
|
||||
//#when
|
||||
const result = getSessionPromptParams(sessionID)!
|
||||
result.temperature = 0.9
|
||||
result.options!.reasoningEffort = "max"
|
||||
|
||||
//#then
|
||||
expect(getSessionPromptParams(sessionID)).toEqual({
|
||||
temperature: 0.2,
|
||||
options: { reasoningEffort: "medium" },
|
||||
})
|
||||
})
|
||||
|
||||
test("clears a single session", () => {
|
||||
//#given
|
||||
const sessionID = "ses_prompt_params_clear"
|
||||
setSessionPromptParams(sessionID, { topP: 0.5 })
|
||||
|
||||
//#when
|
||||
clearSessionPromptParams(sessionID)
|
||||
|
||||
//#then
|
||||
expect(getSessionPromptParams(sessionID)).toBeUndefined()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,34 @@
|
||||
export type SessionPromptParams = {
|
||||
temperature?: number
|
||||
topP?: number
|
||||
options?: Record<string, unknown>
|
||||
}
|
||||
|
||||
const sessionPromptParams = new Map<string, SessionPromptParams>()
|
||||
|
||||
export function setSessionPromptParams(sessionID: string, params: SessionPromptParams): void {
|
||||
sessionPromptParams.set(sessionID, {
|
||||
...(params.temperature !== undefined ? { temperature: params.temperature } : {}),
|
||||
...(params.topP !== undefined ? { topP: params.topP } : {}),
|
||||
...(params.options !== undefined ? { options: { ...params.options } } : {}),
|
||||
})
|
||||
}
|
||||
|
||||
export function getSessionPromptParams(sessionID: string): SessionPromptParams | undefined {
|
||||
const params = sessionPromptParams.get(sessionID)
|
||||
if (!params) return undefined
|
||||
|
||||
return {
|
||||
...(params.temperature !== undefined ? { temperature: params.temperature } : {}),
|
||||
...(params.topP !== undefined ? { topP: params.topP } : {}),
|
||||
...(params.options !== undefined ? { options: { ...params.options } } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
export function clearSessionPromptParams(sessionID: string): void {
|
||||
sessionPromptParams.delete(sessionID)
|
||||
}
|
||||
|
||||
export function clearAllSessionPromptParams(): void {
|
||||
sessionPromptParams.clear()
|
||||
}
|
||||
+13
-1
@@ -1,4 +1,4 @@
|
||||
export type ShellType = "unix" | "powershell" | "cmd"
|
||||
export type ShellType = "unix" | "powershell" | "cmd" | "csh"
|
||||
|
||||
/**
|
||||
* Detect the current shell type based on environment variables.
|
||||
@@ -14,6 +14,10 @@ export function detectShellType(): ShellType {
|
||||
}
|
||||
|
||||
if (process.env.SHELL) {
|
||||
const shell = process.env.SHELL
|
||||
if (shell.includes("csh") || shell.includes("tcsh")) {
|
||||
return "csh"
|
||||
}
|
||||
return "unix"
|
||||
}
|
||||
|
||||
@@ -34,6 +38,7 @@ export function shellEscape(value: string, shellType: ShellType): string {
|
||||
|
||||
switch (shellType) {
|
||||
case "unix":
|
||||
case "csh":
|
||||
if (/[^a-zA-Z0-9_\-.:\/]/.test(value)) {
|
||||
return `'${value.replace(/'/g, "'\\''")}'`
|
||||
}
|
||||
@@ -91,6 +96,13 @@ export function buildEnvPrefix(
|
||||
return `export ${assignments};`
|
||||
}
|
||||
|
||||
case "csh": {
|
||||
const assignments = entries
|
||||
.map(([key, value]) => `setenv ${key} ${shellEscape(value, shellType)}`)
|
||||
.join("; ")
|
||||
return `${assignments};`
|
||||
}
|
||||
|
||||
case "powershell": {
|
||||
const assignments = entries
|
||||
.map(([key, value]) => `$env:${key}=${shellEscape(value, shellType)}`)
|
||||
|
||||
@@ -43,12 +43,12 @@ describe("isInsideTmux", () => {
|
||||
expect(result).toBe(false)
|
||||
})
|
||||
|
||||
test("returns the same result as the process environment helper", () => {
|
||||
test("is exported as a function", () => {
|
||||
// given, #when
|
||||
const result = isInsideTmux()
|
||||
const result = typeof isInsideTmux
|
||||
|
||||
// then
|
||||
expect(result).toBe(isInsideTmuxEnvironment(process.env))
|
||||
expect(result).toBe("function")
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user