Merge pull request #2829 from RaviTharuma/fix/model-capabilities-review-followup

fix(model-capabilities): harden runtime capability handling
2026-03-26 00:25:07 +09:00
parent 5e9231e251 a15f6076bc
commit 76420b36ab
15 changed files with 802 additions and 64 deletions
@@ -0,0 +1,43 @@
+name: Refresh Model Capabilities
+
+on:
+  schedule:
+    - cron: "17 4 * * 1"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  refresh:
+    runs-on: ubuntu-latest
+    if: github.repository == 'code-yeongyu/oh-my-openagent'
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Install dependencies
+        run: bun install
+        env:
+          BUN_INSTALL_ALLOW_SCRIPTS: "@ast-grep/napi"
+
+      - name: Refresh bundled model capabilities snapshot
+        run: bun run build:model-capabilities
+
+      - name: Create refresh pull request
+        uses: peter-evans/create-pull-request@v7
+        with:
+          commit-message: "chore: refresh model capabilities snapshot"
+          title: "chore: refresh model capabilities snapshot"
+          body: |
+            Automated refresh of `src/generated/model-capabilities.generated.json` from `https://models.dev/api.json`.
+
+            This keeps the bundled capability snapshot aligned with upstream model metadata without relying on manual refreshes.
+          branch: automation/refresh-model-capabilities
+          delete-branch: true
+          labels: |
+            maintenance
@@ -4,6 +4,10 @@ import { getOpenCodeCacheDir } from "../../../shared"
 import type { AvailableModelsInfo, ModelResolutionInfo, OmoConfig } from "./model-resolution-types"
 import { formatModelWithVariant, getCategoryEffectiveVariant, getEffectiveVariant } from "./model-resolution-variant"

+function formatCapabilityResolutionLabel(mode: string | undefined): string {
+  return mode ?? "unknown"
+}
+
 export function buildModelResolutionDetails(options: {
  info: ModelResolutionInfo
  available: AvailableModelsInfo
@@ -37,7 +41,7 @@ export function buildModelResolutionDetails(options: {
      agent.effectiveModel,
      getEffectiveVariant(agent.name, agent.requirement, options.config)
    )
-    details.push(`  ${marker} ${agent.name}: ${display}`)
+    details.push(`  ${marker} ${agent.name}: ${display} [capabilities: ${formatCapabilityResolutionLabel(agent.capabilityDiagnostics?.resolutionMode)}]`)
  }
  details.push("")
  details.push("Categories:")
@@ -47,7 +51,7 @@ export function buildModelResolutionDetails(options: {
      category.effectiveModel,
      getCategoryEffectiveVariant(category.name, category.requirement, options.config)
    )
-    details.push(`  ${marker} ${category.name}: ${display}`)
+    details.push(`  ${marker} ${category.name}: ${display} [capabilities: ${formatCapabilityResolutionLabel(category.capabilityDiagnostics?.resolutionMode)}]`)
  }
  details.push("")
  details.push("● = user override, ○ = provider fallback")
@@ -1,3 +1,4 @@
+import type { ModelCapabilitiesDiagnostics } from "../../../shared/model-capabilities"
 import type { ModelRequirement } from "../../../shared/model-requirements"

 export interface AgentResolutionInfo {
@@ -7,6 +8,7 @@ export interface AgentResolutionInfo {
  userVariant?: string
  effectiveModel: string
  effectiveResolution: string
+  capabilityDiagnostics?: ModelCapabilitiesDiagnostics
 }

 export interface CategoryResolutionInfo {
@@ -16,6 +18,7 @@ export interface CategoryResolutionInfo {
  userVariant?: string
  effectiveModel: string
  effectiveResolution: string
+  capabilityDiagnostics?: ModelCapabilitiesDiagnostics
 }

 export interface ModelResolutionInfo {
@@ -129,6 +129,19 @@ describe("model-resolution check", () => {
      expect(visual!.userOverride).toBe("google/gemini-3-flash-preview")
      expect(visual!.userVariant).toBe("high")
    })
+
+    it("attaches snapshot-backed capability diagnostics for built-in models", async () => {
+      const { getModelResolutionInfoWithOverrides } = await import("./model-resolution")
+
+      const info = getModelResolutionInfoWithOverrides({})
+      const sisyphus = info.agents.find((a) => a.name === "sisyphus")
+
+      expect(sisyphus).toBeDefined()
+      expect(sisyphus!.capabilityDiagnostics).toMatchObject({
+        resolutionMode: "snapshot-backed",
+        snapshot: { source: "bundled-snapshot" },
+      })
+    })
  })

  describe("checkModelResolution", () => {
@@ -162,6 +175,23 @@ describe("model-resolution check", () => {
      expect(result.details!.some((d) => d.includes("Categories:"))).toBe(true)
      // Should have legend
      expect(result.details!.some((d) => d.includes("user override"))).toBe(true)
+      expect(result.details!.some((d) => d.includes("capabilities: snapshot-backed"))).toBe(true)
+    })
+
+    it("collects warnings when configured models rely on compatibility fallback", async () => {
+      const { collectCapabilityResolutionIssues, getModelResolutionInfoWithOverrides } = await import("./model-resolution")
+
+      const info = getModelResolutionInfoWithOverrides({
+        agents: {
+          oracle: { model: "custom/unknown-llm" },
+        },
+      })
+
+      const issues = collectCapabilityResolutionIssues(info)
+
+      expect(issues).toHaveLength(1)
+      expect(issues[0]?.title).toContain("compatibility fallback")
+      expect(issues[0]?.description).toContain("oracle=custom/unknown-llm")
    })
  })

@@ -1,4 +1,5 @@
 import { AGENT_MODEL_REQUIREMENTS, CATEGORY_MODEL_REQUIREMENTS } from "../../../shared/model-requirements"
+import { getModelCapabilities } from "../../../shared/model-capabilities"
 import { CHECK_IDS, CHECK_NAMES } from "../constants"
 import type { CheckResult, DoctorIssue } from "../types"
 import { loadAvailableModelsFromCache } from "./model-resolution-cache"
@@ -7,16 +8,36 @@ import { buildModelResolutionDetails } from "./model-resolution-details"
 import { buildEffectiveResolution, getEffectiveModel } from "./model-resolution-effective-model"
 import type { AgentResolutionInfo, CategoryResolutionInfo, ModelResolutionInfo, OmoConfig } from "./model-resolution-types"

-export function getModelResolutionInfo(): ModelResolutionInfo {
-  const agents: AgentResolutionInfo[] = Object.entries(AGENT_MODEL_REQUIREMENTS).map(([name, requirement]) => ({
-    name,
-    requirement,
-    effectiveModel: getEffectiveModel(requirement),
-    effectiveResolution: buildEffectiveResolution(requirement),
-  }))
+function parseProviderModel(value: string): { providerID: string; modelID: string } | null {
+  const slashIndex = value.indexOf("/")
+  if (slashIndex <= 0 || slashIndex === value.length - 1) {
+    return null
+  }

-  const categories: CategoryResolutionInfo[] = Object.entries(CATEGORY_MODEL_REQUIREMENTS).map(
-    ([name, requirement]) => ({
+  return {
+    providerID: value.slice(0, slashIndex),
+    modelID: value.slice(slashIndex + 1),
+  }
+}
+
+function attachCapabilityDiagnostics<T extends AgentResolutionInfo | CategoryResolutionInfo>(entry: T): T {
+  const parsed = parseProviderModel(entry.effectiveModel)
+  if (!parsed) {
+    return entry
+  }
+
+  return {
+    ...entry,
+    capabilityDiagnostics: getModelCapabilities({
+      providerID: parsed.providerID,
+      modelID: parsed.modelID,
+    }).diagnostics,
+  }
+}
+
+export function getModelResolutionInfo(): ModelResolutionInfo {
+  const agents: AgentResolutionInfo[] = Object.entries(AGENT_MODEL_REQUIREMENTS).map(([name, requirement]) =>
+    attachCapabilityDiagnostics({
      name,
      requirement,
      effectiveModel: getEffectiveModel(requirement),
@@ -24,6 +45,16 @@ export function getModelResolutionInfo(): ModelResolutionInfo {
    })
  )

+  const categories: CategoryResolutionInfo[] = Object.entries(CATEGORY_MODEL_REQUIREMENTS).map(
+    ([name, requirement]) =>
+      attachCapabilityDiagnostics({
+        name,
+        requirement,
+        effectiveModel: getEffectiveModel(requirement),
+        effectiveResolution: buildEffectiveResolution(requirement),
+      })
+  )
+
  return { agents, categories }
 }

@@ -31,34 +62,60 @@ export function getModelResolutionInfoWithOverrides(config: OmoConfig): ModelRes
  const agents: AgentResolutionInfo[] = Object.entries(AGENT_MODEL_REQUIREMENTS).map(([name, requirement]) => {
    const userOverride = config.agents?.[name]?.model
    const userVariant = config.agents?.[name]?.variant
-    return {
+    return attachCapabilityDiagnostics({
      name,
      requirement,
      userOverride,
      userVariant,
      effectiveModel: getEffectiveModel(requirement, userOverride),
      effectiveResolution: buildEffectiveResolution(requirement, userOverride),
-    }
+    })
  })

  const categories: CategoryResolutionInfo[] = Object.entries(CATEGORY_MODEL_REQUIREMENTS).map(
    ([name, requirement]) => {
      const userOverride = config.categories?.[name]?.model
      const userVariant = config.categories?.[name]?.variant
-      return {
+      return attachCapabilityDiagnostics({
        name,
        requirement,
        userOverride,
        userVariant,
        effectiveModel: getEffectiveModel(requirement, userOverride),
        effectiveResolution: buildEffectiveResolution(requirement, userOverride),
-      }
+      })
    }
  )

  return { agents, categories }
 }

+export function collectCapabilityResolutionIssues(info: ModelResolutionInfo): DoctorIssue[] {
+  const issues: DoctorIssue[] = []
+  const allEntries = [...info.agents, ...info.categories]
+  const fallbackEntries = allEntries.filter((entry) => {
+    const mode = entry.capabilityDiagnostics?.resolutionMode
+    return mode === "alias-backed" || mode === "heuristic-backed" || mode === "unknown"
+  })
+
+  if (fallbackEntries.length === 0) {
+    return issues
+  }
+
+  const summary = fallbackEntries
+    .map((entry) => `${entry.name}=${entry.effectiveModel} (${entry.capabilityDiagnostics?.resolutionMode ?? "unknown"})`)
+    .join(", ")
+
+  issues.push({
+    title: "Configured models rely on compatibility fallback",
+    description: summary,
+    severity: "warning",
+    affects: fallbackEntries.map((entry) => entry.name),
+  })
+
+  return issues
+}
+
 export async function checkModels(): Promise<CheckResult> {
  const config = loadOmoConfig() ?? {}
  const info = getModelResolutionInfoWithOverrides(config)
@@ -75,6 +132,8 @@ export async function checkModels(): Promise<CheckResult> {
    })
  }

+  issues.push(...collectCapabilityResolutionIssues(info))
+
  const overrideCount =
    info.agents.filter((agent) => Boolean(agent.userOverride)).length +
    info.categories.filter((category) => Boolean(category.userOverride)).length
@@ -229,4 +229,38 @@ describe("updateConnectedProvidersCache", () => {
 			limit: { output: 128000 },
 		})
 	})
+
+	test("keeps normalized fallback ids when raw metadata id is not a string", async () => {
+		const mockClient = {
+			provider: {
+				list: async () => ({
+					data: {
+						connected: ["openai"],
+						all: [
+							{
+								id: "openai",
+								models: {
+									"o3-mini": {
+										id: 123,
+										name: "o3-mini",
+									},
+								},
+							},
+						],
+					},
+				}),
+			},
+		}
+
+		await testCacheStore.updateConnectedProvidersCache(mockClient)
+		const cache = testCacheStore.readProviderModelsCache()
+
+		expect(cache?.models.openai).toEqual([
+			{ id: "o3-mini", name: "o3-mini" },
+		])
+		expect(findProviderModelMetadata("openai", "o3-mini", cache)).toEqual({
+			id: "o3-mini",
+			name: "o3-mini",
+		})
+	})
 })
@@ -198,8 +198,8 @@ export function createConnectedProvidersCacheStore(
 							: modelID

 						return {
-							id: normalizedID,
 							...rawMetadata,
+							id: normalizedID,
 						} satisfies ModelMetadata
 					})
 					if (modelMetadata.length > 0) {
@@ -97,6 +97,37 @@ describe("model-capabilities-cache", () => {
    })
  })

+  test("merges repeated snapshot entries without materializing empty optional objects", () => {
+    const raw = {
+      openai: {
+        models: {
+          "gpt-5.4": {
+            id: "gpt-5.4",
+            family: "gpt",
+          },
+        },
+      },
+      alias: {
+        models: {
+          "gpt-5.4-preview": {
+            id: "gpt-5.4",
+            reasoning: true,
+          },
+        },
+      },
+    }
+
+    const snapshot = buildModelCapabilitiesSnapshotFromModelsDev(raw)
+
+    expect(snapshot.models["gpt-5.4"]).toEqual({
+      id: "gpt-5.4",
+      family: "gpt",
+      reasoning: true,
+    })
+    expect(snapshot.models["gpt-5.4"]).not.toHaveProperty("modalities")
+    expect(snapshot.models["gpt-5.4"]).not.toHaveProperty("limit")
+  })
+
  test("refresh writes cache and preserves unrelated files in the cache directory", async () => {
    //#given
    const sentinelPath = join(testCacheDir, "keep-me.json")
@@ -8,7 +8,7 @@ export const MODELS_DEV_SOURCE_URL = "https://models.dev/api.json"
 const MODEL_CAPABILITIES_CACHE_FILE = "model-capabilities.json"

 function isRecord(value: unknown): value is Record<string, unknown> {
-  return typeof value === "object" && value !== null
+  return typeof value === "object" && value !== null && !Array.isArray(value)
 }

 function readBoolean(value: unknown): boolean | undefined {
@@ -84,17 +84,24 @@ function mergeSnapshotEntries(
    return incoming
  }

+  const mergedModalities = existing.modalities || incoming.modalities
+    ? {
+        ...existing.modalities,
+        ...incoming.modalities,
+      }
+    : undefined
+  const mergedLimit = existing.limit || incoming.limit
+    ? {
+        ...existing.limit,
+        ...incoming.limit,
+      }
+    : undefined
+
  return {
    ...existing,
    ...incoming,
-    modalities: {
-      ...existing.modalities,
-      ...incoming.modalities,
-    },
-    limit: {
-      ...existing.limit,
-      ...incoming.limit,
-    },
+    ...(mergedModalities ? { modalities: mergedModalities } : {}),
+    ...(mergedLimit ? { limit: mergedLimit } : {}),
  }
 }

@@ -2,8 +2,10 @@ import { describe, expect, test } from "bun:test"

 import {
  getModelCapabilities,
+  getBundledModelCapabilitiesSnapshot,
  type ModelCapabilitiesSnapshot,
 } from "./model-capabilities"
+import { AGENT_MODEL_REQUIREMENTS, CATEGORY_MODEL_REQUIREMENTS } from "./model-requirements"

 describe("getModelCapabilities", () => {
  const bundledSnapshot: ModelCapabilitiesSnapshot = {
@@ -79,6 +81,84 @@ describe("getModelCapabilities", () => {
      maxOutputTokens: 128_000,
      toolCall: true,
    })
+    expect(result.diagnostics).toMatchObject({
+      resolutionMode: "snapshot-backed",
+      canonicalization: { source: "canonical" },
+      snapshot: { source: "bundled-snapshot" },
+      variants: { source: "runtime" },
+    })
+  })
+
+  test("reads structured runtime capabilities from the SDK v2 shape", () => {
+    const result = getModelCapabilities({
+      providerID: "openai",
+      modelID: "gpt-5.4",
+      runtimeModel: {
+        capabilities: {
+          reasoning: true,
+          temperature: false,
+          toolcall: true,
+          input: {
+            text: true,
+            image: true,
+          },
+          output: {
+            text: true,
+          },
+        },
+      },
+      bundledSnapshot,
+    })
+
+    expect(result).toMatchObject({
+      canonicalModelID: "gpt-5.4",
+      reasoning: true,
+      supportsThinking: true,
+      supportsTemperature: false,
+      toolCall: true,
+      modalities: {
+        input: ["text", "image"],
+        output: ["text"],
+      },
+    })
+    expect(result.diagnostics).toMatchObject({
+      resolutionMode: "snapshot-backed",
+      reasoning: { source: "runtime" },
+      supportsThinking: { source: "runtime" },
+      toolCall: { source: "runtime" },
+    })
+  })
+
+  test("respects root-level thinking flags when providers do not nest them under capabilities", () => {
+    const result = getModelCapabilities({
+      providerID: "custom-proxy",
+      modelID: "gpt-5.4",
+      runtimeModel: {
+        supportsThinking: true,
+      },
+      bundledSnapshot,
+    })
+
+    expect(result).toMatchObject({
+      canonicalModelID: "gpt-5.4",
+      supportsThinking: true,
+    })
+    expect(result.diagnostics).toMatchObject({
+      supportsThinking: { source: "runtime" },
+    })
+  })
+
+  test("accepts runtime variant arrays without corrupting them into numeric keys", () => {
+    const result = getModelCapabilities({
+      providerID: "openai",
+      modelID: "gpt-5.4",
+      runtimeModel: {
+        variants: ["low", "medium", "high", "xhigh"],
+      },
+      bundledSnapshot,
+    })
+
+    expect(result.variants).toEqual(["low", "medium", "high", "xhigh"])
  })

  test("normalizes thinking suffix aliases before snapshot lookup", () => {
@@ -95,6 +175,14 @@ describe("getModelCapabilities", () => {
      supportsTemperature: true,
      maxOutputTokens: 128_000,
    })
+    expect(result.diagnostics).toMatchObject({
+      resolutionMode: "alias-backed",
+      canonicalization: {
+        source: "pattern-alias",
+        ruleID: "anthropic-thinking-suffix",
+      },
+      snapshot: { source: "bundled-snapshot" },
+    })
  })

  test("maps local gemini aliases to canonical models.dev entries", () => {
@@ -111,6 +199,14 @@ describe("getModelCapabilities", () => {
      supportsTemperature: true,
      maxOutputTokens: 65_000,
    })
+    expect(result.diagnostics).toMatchObject({
+      resolutionMode: "alias-backed",
+      canonicalization: {
+        source: "exact-alias",
+        ruleID: "gemini-3.1-pro-tier-alias",
+      },
+      snapshot: { source: "bundled-snapshot" },
+    })
  })

  test("prefers runtime models.dev cache over bundled snapshot", () => {
@@ -140,6 +236,11 @@ describe("getModelCapabilities", () => {
      maxOutputTokens: 64_000,
      supportsTemperature: false,
    })
+    expect(result.diagnostics).toMatchObject({
+      snapshot: { source: "runtime-snapshot" },
+      maxOutputTokens: { source: "runtime-snapshot" },
+      supportsTemperature: { source: "runtime-snapshot" },
+    })
  })

  test("falls back to heuristic family rules when no snapshot entry exists", () => {
@@ -155,5 +256,55 @@ describe("getModelCapabilities", () => {
      variants: ["low", "medium", "high"],
      reasoningEfforts: ["none", "minimal", "low", "medium", "high"],
    })
+    expect(result.diagnostics).toMatchObject({
+      resolutionMode: "heuristic-backed",
+      snapshot: { source: "none" },
+      family: { source: "heuristic" },
+      reasoningEfforts: { source: "heuristic" },
+    })
+  })
+
+  test("detects prefixed o-series model IDs through the heuristic fallback", () => {
+    const result = getModelCapabilities({
+      providerID: "azure-openai",
+      modelID: "openai/o3-mini",
+      bundledSnapshot,
+    })
+
+    expect(result).toMatchObject({
+      canonicalModelID: "openai/o3-mini",
+      family: "openai-reasoning",
+      variants: ["low", "medium", "high"],
+      reasoningEfforts: ["none", "minimal", "low", "medium", "high"],
+    })
+    expect(result.diagnostics).toMatchObject({
+      resolutionMode: "heuristic-backed",
+      snapshot: { source: "none" },
+      family: { source: "heuristic" },
+    })
+  })
+
+  test("keeps every built-in OmO requirement model snapshot-backed", () => {
+    const bundledSnapshot = getBundledModelCapabilitiesSnapshot()
+    const requirementModels = new Set<string>()
+
+    for (const requirement of Object.values(AGENT_MODEL_REQUIREMENTS)) {
+      for (const entry of requirement.fallbackChain) requirementModels.add(entry.model)
+    }
+
+    for (const requirement of Object.values(CATEGORY_MODEL_REQUIREMENTS)) {
+      for (const entry of requirement.fallbackChain) requirementModels.add(entry.model)
+    }
+
+    for (const modelID of requirementModels) {
+      const result = getModelCapabilities({
+        providerID: "test-provider",
+        modelID,
+        bundledSnapshot,
+      })
+
+      expect(result.diagnostics.resolutionMode).toBe("snapshot-backed")
+      expect(result.diagnostics.snapshot.source).toBe("bundled-snapshot")
+    }
  })
 })
@@ -1,5 +1,6 @@
 import bundledModelCapabilitiesSnapshotJson from "../generated/model-capabilities.generated.json"
 import { findProviderModelMetadata, type ModelMetadata } from "./connected-providers-cache"
+import { resolveModelIDAlias } from "./model-capability-aliases"
 import { detectHeuristicModelFamily } from "./model-capability-heuristics"

 export type ModelCapabilitiesSnapshotEntry = {
@@ -41,6 +42,7 @@ export type ModelCapabilities = {
    input?: string[]
    output?: string[]
  }
+  diagnostics: ModelCapabilitiesDiagnostics
 }

 type GetModelCapabilitiesInput = {
@@ -52,7 +54,6 @@ type GetModelCapabilitiesInput = {
 }

 type ModelCapabilityOverride = {
-  canonicalModelID?: string
  variants?: string[]
  reasoningEfforts?: string[]
  supportsThinking?: boolean
@@ -60,19 +61,42 @@ type ModelCapabilityOverride = {
  supportsTopP?: boolean
 }

-const MODEL_ID_OVERRIDES: Record<string, ModelCapabilityOverride> = {
-  "claude-opus-4-6-thinking": { canonicalModelID: "claude-opus-4-6" },
-  "claude-sonnet-4-6-thinking": { canonicalModelID: "claude-sonnet-4-6" },
-  "claude-opus-4-5-thinking": { canonicalModelID: "claude-opus-4-5-20251101" },
-  "gpt-5.3-codex-spark": { canonicalModelID: "gpt-5.3-codex" },
-  "gemini-3.1-pro-high": { canonicalModelID: "gemini-3.1-pro-preview" },
-  "gemini-3.1-pro-low": { canonicalModelID: "gemini-3.1-pro-preview" },
-  "gemini-3-pro-high": { canonicalModelID: "gemini-3-pro-preview" },
-  "gemini-3-pro-low": { canonicalModelID: "gemini-3-pro-preview" },
+type DiagnosticSource =
+  | "none"
+  | "runtime"
+  | "runtime-snapshot"
+  | "bundled-snapshot"
+  | "override"
+  | "heuristic"
+  | "canonical"
+  | "exact-alias"
+  | "pattern-alias"
+
+export type ModelCapabilitiesDiagnostics = {
+  resolutionMode: "snapshot-backed" | "alias-backed" | "heuristic-backed" | "unknown"
+  canonicalization: {
+    source: "canonical" | "exact-alias" | "pattern-alias"
+    ruleID?: string
+  }
+  snapshot: {
+    source: "runtime-snapshot" | "bundled-snapshot" | "none"
+  }
+  family: { source: "snapshot" | "heuristic" | "none" }
+  variants: { source: Exclude<DiagnosticSource, "runtime-snapshot" | "bundled-snapshot" | "exact-alias" | "pattern-alias"> }
+  reasoningEfforts: { source: Exclude<DiagnosticSource, "runtime-snapshot" | "bundled-snapshot" | "canonical" | "exact-alias" | "pattern-alias" | "runtime"> }
+  reasoning: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" }
+  supportsThinking: { source: "runtime" | "override" | "heuristic" | "runtime-snapshot" | "bundled-snapshot" | "none" }
+  supportsTemperature: { source: "runtime" | "override" | "runtime-snapshot" | "bundled-snapshot" | "none" }
+  supportsTopP: { source: "runtime" | "override" | "none" }
+  maxOutputTokens: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" }
+  toolCall: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" }
+  modalities: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" }
 }

+const MODEL_ID_OVERRIDES: Record<string, ModelCapabilityOverride> = {}
+
 function isRecord(value: unknown): value is Record<string, unknown> {
-  return typeof value === "object" && value !== null
+  return typeof value === "object" && value !== null && !Array.isArray(value)
 }

 function normalizeLookupModelID(modelID: string): string {
@@ -97,6 +121,11 @@ function readStringArray(value: unknown): string[] | undefined {
 }

 function normalizeVariantKeys(value: unknown): string[] | undefined {
+  const arrayVariants = readStringArray(value)
+  if (arrayVariants) {
+    return arrayVariants.map((variant) => variant.toLowerCase())
+  }
+
  if (!isRecord(value)) {
    return undefined
  }
@@ -105,13 +134,30 @@ function normalizeVariantKeys(value: unknown): string[] | undefined {
  return variants.length > 0 ? variants : undefined
 }

+function readModalityKeys(value: unknown): string[] | undefined {
+  const stringArray = readStringArray(value)
+  if (stringArray) {
+    return stringArray.map((entry) => entry.toLowerCase())
+  }
+
+  if (!isRecord(value)) {
+    return undefined
+  }
+
+  const enabled = Object.entries(value)
+    .filter(([, supported]) => supported === true)
+    .map(([modality]) => modality.toLowerCase())
+
+  return enabled.length > 0 ? enabled : undefined
+}
+
 function normalizeModalities(value: unknown): ModelCapabilities["modalities"] | undefined {
  if (!isRecord(value)) {
    return undefined
  }

-  const input = readStringArray(value.input)
-  const output = readStringArray(value.output)
+  const input = readModalityKeys(value.input)
+  const output = readModalityKeys(value.output)

  if (!input && !output) {
    return undefined
@@ -127,30 +173,22 @@ function normalizeSnapshot(snapshot: ModelCapabilitiesSnapshot | typeof bundledM
  return snapshot as ModelCapabilitiesSnapshot
 }

-function getCanonicalModelID(modelID: string): string {
-  const normalizedModelID = normalizeLookupModelID(modelID)
-  const override = MODEL_ID_OVERRIDES[normalizedModelID]
-  if (override?.canonicalModelID) {
-    return override.canonicalModelID
-  }
-
-  if (normalizedModelID.startsWith("claude-") && normalizedModelID.endsWith("-thinking")) {
-    return normalizedModelID.replace(/-thinking$/i, "")
-  }
-
-  return normalizedModelID
-}
-
 function getOverride(modelID: string): ModelCapabilityOverride | undefined {
  return MODEL_ID_OVERRIDES[normalizeLookupModelID(modelID)]
 }

+function readRuntimeModelCapabilities(runtimeModel: Record<string, unknown> | undefined): Record<string, unknown> | undefined {
+  return isRecord(runtimeModel?.capabilities) ? runtimeModel.capabilities : undefined
+}
+
 function readRuntimeModelLimitOutput(runtimeModel: Record<string, unknown> | undefined): number | undefined {
  if (!runtimeModel) {
    return undefined
  }

-  const limit = runtimeModel.limit
+  const limit = isRecord(runtimeModel.limit)
+    ? runtimeModel.limit
+    : readRuntimeModelCapabilities(runtimeModel)?.limit
  if (!isRecord(limit)) {
    return undefined
  }
@@ -163,11 +201,106 @@ function readRuntimeModelBoolean(runtimeModel: Record<string, unknown> | undefin
    return undefined
  }

+  const runtimeCapabilities = readRuntimeModelCapabilities(runtimeModel)
+
  for (const key of keys) {
    const value = runtimeModel[key]
    if (typeof value === "boolean") {
      return value
    }
+
+    const capabilityValue = runtimeCapabilities?.[key]
+    if (typeof capabilityValue === "boolean") {
+      return capabilityValue
+    }
+  }
+
+  return undefined
+}
+
+function readRuntimeModelModalities(runtimeModel: Record<string, unknown> | undefined): ModelCapabilities["modalities"] | undefined {
+  if (!runtimeModel) {
+    return undefined
+  }
+
+  const rootModalities = normalizeModalities(runtimeModel.modalities)
+  if (rootModalities) {
+    return rootModalities
+  }
+
+  const runtimeCapabilities = readRuntimeModelCapabilities(runtimeModel)
+  if (!runtimeCapabilities) {
+    return undefined
+  }
+
+  const nestedModalities = normalizeModalities(runtimeCapabilities.modalities)
+  if (nestedModalities) {
+    return nestedModalities
+  }
+
+  const capabilityModalities = normalizeModalities(runtimeCapabilities)
+  if (capabilityModalities) {
+    return capabilityModalities
+  }
+
+  return undefined
+}
+
+function readRuntimeModelVariants(runtimeModel: Record<string, unknown> | undefined): string[] | undefined {
+  if (!runtimeModel) {
+    return undefined
+  }
+
+  const rootVariants = normalizeVariantKeys(runtimeModel.variants)
+  if (rootVariants) {
+    return rootVariants
+  }
+
+  const runtimeCapabilities = readRuntimeModelCapabilities(runtimeModel)
+  if (!runtimeCapabilities) {
+    return undefined
+  }
+
+  return normalizeVariantKeys(runtimeCapabilities.variants)
+}
+
+function readRuntimeModelTopPSupport(runtimeModel: Record<string, unknown> | undefined): boolean | undefined {
+  return readRuntimeModelBoolean(runtimeModel, ["topP", "top_p"])
+}
+
+function readRuntimeModelToolCallSupport(runtimeModel: Record<string, unknown> | undefined): boolean | undefined {
+  return readRuntimeModelBoolean(runtimeModel, ["toolCall", "tool_call", "toolcall"])
+}
+
+function readRuntimeModelReasoningSupport(runtimeModel: Record<string, unknown> | undefined): boolean | undefined {
+  return readRuntimeModelBoolean(runtimeModel, ["reasoning"])
+}
+
+function readRuntimeModelTemperatureSupport(runtimeModel: Record<string, unknown> | undefined): boolean | undefined {
+  return readRuntimeModelBoolean(runtimeModel, ["temperature"])
+}
+
+function readRuntimeModelThinkingSupport(runtimeModel: Record<string, unknown> | undefined): boolean | undefined {
+  const capabilityValue = readRuntimeModelReasoningSupport(runtimeModel)
+  if (capabilityValue !== undefined) {
+    return capabilityValue
+  }
+
+  const rootThinkingSupport = readRuntimeModelBoolean(runtimeModel, ["thinking", "supportsThinking"])
+  if (rootThinkingSupport !== undefined) {
+    return rootThinkingSupport
+  }
+
+  const runtimeCapabilities = readRuntimeModelCapabilities(runtimeModel)
+  if (!runtimeCapabilities) {
+    return undefined
+  }
+
+  for (const key of ["thinking", "supportsThinking"] as const) {
+    const value = runtimeCapabilities[key]
+    if (typeof value === "boolean") {
+      return value
+    }
  }

  return undefined
@@ -184,8 +317,9 @@ export function getBundledModelCapabilitiesSnapshot(): ModelCapabilitiesSnapshot
 }

 export function getModelCapabilities(input: GetModelCapabilitiesInput): ModelCapabilities {
-  const requestedModelID = normalizeLookupModelID(input.modelID)
-  const canonicalModelID = getCanonicalModelID(input.modelID)
+  const canonicalization = resolveModelIDAlias(input.modelID)
+  const requestedModelID = canonicalization.requestedModelID
+  const canonicalModelID = canonicalization.canonicalModelID
  const override = getOverride(input.modelID)
  const runtimeModel = readRuntimeModel(
    input.runtimeModel ?? findProviderModelMetadata(input.providerID, input.modelID),
@@ -194,7 +328,89 @@ export function getModelCapabilities(input: GetModelCapabilitiesInput): ModelCap
  const bundledSnapshot = input.bundledSnapshot ?? bundledModelCapabilitiesSnapshot
  const snapshotEntry = runtimeSnapshot?.models?.[canonicalModelID] ?? bundledSnapshot.models[canonicalModelID]
  const heuristicFamily = detectHeuristicModelFamily(canonicalModelID)
-  const runtimeVariants = normalizeVariantKeys(runtimeModel?.variants)
+  const runtimeVariants = readRuntimeModelVariants(runtimeModel)
+  const snapshotSource: ModelCapabilitiesDiagnostics["snapshot"]["source"] =
+    runtimeSnapshot?.models?.[canonicalModelID]
+      ? "runtime-snapshot"
+      : bundledSnapshot.models[canonicalModelID]
+      ? "bundled-snapshot"
+      : "none"
+  const familySource: ModelCapabilitiesDiagnostics["family"]["source"] =
+    snapshotEntry?.family
+      ? "snapshot"
+      : heuristicFamily?.family
+      ? "heuristic"
+      : "none"
+  const variantsSource: ModelCapabilitiesDiagnostics["variants"]["source"] =
+    runtimeVariants
+      ? "runtime"
+      : override?.variants
+      ? "override"
+      : heuristicFamily?.variants
+      ? "heuristic"
+      : "none"
+  const reasoningEffortsSource: ModelCapabilitiesDiagnostics["reasoningEfforts"]["source"] =
+    override?.reasoningEfforts
+      ? "override"
+      : heuristicFamily?.reasoningEfforts
+      ? "heuristic"
+      : "none"
+  const reasoningSource: ModelCapabilitiesDiagnostics["reasoning"]["source"] =
+    readRuntimeModelReasoningSupport(runtimeModel) !== undefined
+      ? "runtime"
+      : snapshotEntry?.reasoning !== undefined
+      ? snapshotSource
+      : "none"
+  const supportsThinkingSource: ModelCapabilitiesDiagnostics["supportsThinking"]["source"] =
+    override?.supportsThinking !== undefined
+      ? "override"
+      : heuristicFamily?.supportsThinking !== undefined
+      ? "heuristic"
+      : readRuntimeModelThinkingSupport(runtimeModel) !== undefined
+      ? "runtime"
+      : snapshotEntry?.reasoning !== undefined
+      ? snapshotSource
+      : "none"
+  const supportsTemperatureSource: ModelCapabilitiesDiagnostics["supportsTemperature"]["source"] =
+    readRuntimeModelTemperatureSupport(runtimeModel) !== undefined
+      ? "runtime"
+      : override?.supportsTemperature !== undefined
+      ? "override"
+      : snapshotEntry?.temperature !== undefined
+      ? snapshotSource
+      : "none"
+  const supportsTopPSource: ModelCapabilitiesDiagnostics["supportsTopP"]["source"] =
+    readRuntimeModelTopPSupport(runtimeModel) !== undefined
+      ? "runtime"
+      : override?.supportsTopP !== undefined
+      ? "override"
+      : "none"
+  const maxOutputTokensSource: ModelCapabilitiesDiagnostics["maxOutputTokens"]["source"] =
+    readRuntimeModelLimitOutput(runtimeModel) !== undefined
+      ? "runtime"
+      : snapshotEntry?.limit?.output !== undefined
+      ? snapshotSource
+      : "none"
+  const toolCallSource: ModelCapabilitiesDiagnostics["toolCall"]["source"] =
+    readRuntimeModelToolCallSupport(runtimeModel) !== undefined
+      ? "runtime"
+      : snapshotEntry?.toolCall !== undefined
+      ? snapshotSource
+      : "none"
+  const modalitiesSource: ModelCapabilitiesDiagnostics["modalities"]["source"] =
+    readRuntimeModelModalities(runtimeModel) !== undefined
+      ? "runtime"
+      : snapshotEntry?.modalities !== undefined
+      ? snapshotSource
+      : "none"
+  const resolutionMode: ModelCapabilitiesDiagnostics["resolutionMode"] =
+    snapshotSource !== "none" && canonicalization.source === "canonical"
+      ? "snapshot-backed"
+      : snapshotSource !== "none"
+      ? "alias-backed"
+      : familySource === "heuristic" || variantsSource === "heuristic" || reasoningEffortsSource === "heuristic"
+      ? "heuristic-backed"
+      : "unknown"

  return {
    requestedModelID,
@@ -202,27 +418,45 @@ export function getModelCapabilities(input: GetModelCapabilitiesInput): ModelCap
    family: snapshotEntry?.family ?? heuristicFamily?.family,
    variants: runtimeVariants ?? override?.variants ?? heuristicFamily?.variants,
    reasoningEfforts: override?.reasoningEfforts ?? heuristicFamily?.reasoningEfforts,
-    reasoning: readRuntimeModelBoolean(runtimeModel, ["reasoning"]) ?? snapshotEntry?.reasoning,
+    reasoning: readRuntimeModelReasoningSupport(runtimeModel) ?? snapshotEntry?.reasoning,
    supportsThinking:
      override?.supportsThinking
      ?? heuristicFamily?.supportsThinking
-      ?? readRuntimeModelBoolean(runtimeModel, ["reasoning"])
+      ?? readRuntimeModelThinkingSupport(runtimeModel)
      ?? snapshotEntry?.reasoning,
    supportsTemperature:
-      readRuntimeModelBoolean(runtimeModel, ["temperature"])
+      readRuntimeModelTemperatureSupport(runtimeModel)
      ?? override?.supportsTemperature
      ?? snapshotEntry?.temperature,
    supportsTopP:
-      readRuntimeModelBoolean(runtimeModel, ["topP", "top_p"])
+      readRuntimeModelTopPSupport(runtimeModel)
      ?? override?.supportsTopP,
    maxOutputTokens:
      readRuntimeModelLimitOutput(runtimeModel)
      ?? snapshotEntry?.limit?.output,
    toolCall:
-      readRuntimeModelBoolean(runtimeModel, ["toolCall", "tool_call"])
+      readRuntimeModelToolCallSupport(runtimeModel)
      ?? snapshotEntry?.toolCall,
    modalities:
-      normalizeModalities(runtimeModel?.modalities)
+      readRuntimeModelModalities(runtimeModel)
      ?? snapshotEntry?.modalities,
+    diagnostics: {
+      resolutionMode,
+      canonicalization: {
+        source: canonicalization.source,
+        ...(canonicalization.ruleID ? { ruleID: canonicalization.ruleID } : {}),
+      },
+      snapshot: { source: snapshotSource },
+      family: { source: familySource },
+      variants: { source: variantsSource },
+      reasoningEfforts: { source: reasoningEffortsSource },
+      reasoning: { source: reasoningSource },
+      supportsThinking: { source: supportsThinkingSource },
+      supportsTemperature: { source: supportsTemperatureSource },
+      supportsTopP: { source: supportsTopPSource },
+      maxOutputTokens: { source: maxOutputTokensSource },
+      toolCall: { source: toolCallSource },
+      modalities: { source: modalitiesSource },
+    },
  }
 }
@@ -0,0 +1,37 @@
+import { describe, expect, test } from "bun:test"
+
+import { resolveModelIDAlias } from "./model-capability-aliases"
+
+describe("model-capability-aliases", () => {
+  test("keeps canonical model IDs unchanged", () => {
+    const result = resolveModelIDAlias("gpt-5.4")
+
+    expect(result).toEqual({
+      requestedModelID: "gpt-5.4",
+      canonicalModelID: "gpt-5.4",
+      source: "canonical",
+    })
+  })
+
+  test("normalizes exact local tier aliases to canonical models.dev IDs", () => {
+    const result = resolveModelIDAlias("gemini-3.1-pro-high")
+
+    expect(result).toEqual({
+      requestedModelID: "gemini-3.1-pro-high",
+      canonicalModelID: "gemini-3.1-pro-preview",
+      source: "exact-alias",
+      ruleID: "gemini-3.1-pro-tier-alias",
+    })
+  })
+
+  test("normalizes decorated thinking aliases through a named pattern rule", () => {
+    const result = resolveModelIDAlias("claude-opus-4-6-thinking")
+
+    expect(result).toEqual({
+      requestedModelID: "claude-opus-4-6-thinking",
+      canonicalModelID: "claude-opus-4-6",
+      source: "pattern-alias",
+      ruleID: "anthropic-thinking-suffix",
+    })
+  })
+})
@@ -0,0 +1,84 @@
+type ExactAliasRule = {
+  ruleID: string
+  canonicalModelID: string
+}
+
+type PatternAliasRule = {
+  ruleID: string
+  match: (normalizedModelID: string) => boolean
+  canonicalize: (normalizedModelID: string) => string
+}
+
+export type ModelIDAliasResolution = {
+  requestedModelID: string
+  canonicalModelID: string
+  source: "canonical" | "exact-alias" | "pattern-alias"
+  ruleID?: string
+}
+
+const EXACT_ALIAS_RULES: Record<string, ExactAliasRule> = {
+  "gpt-5.3-codex-spark": {
+    ruleID: "gpt-5.3-codex-spark-alias",
+    canonicalModelID: "gpt-5.3-codex",
+  },
+  "gemini-3.1-pro-high": {
+    ruleID: "gemini-3.1-pro-tier-alias",
+    canonicalModelID: "gemini-3.1-pro-preview",
+  },
+  "gemini-3.1-pro-low": {
+    ruleID: "gemini-3.1-pro-tier-alias",
+    canonicalModelID: "gemini-3.1-pro-preview",
+  },
+  "gemini-3-pro-high": {
+    ruleID: "gemini-3-pro-tier-alias",
+    canonicalModelID: "gemini-3-pro-preview",
+  },
+  "gemini-3-pro-low": {
+    ruleID: "gemini-3-pro-tier-alias",
+    canonicalModelID: "gemini-3-pro-preview",
+  },
+}
+
+const PATTERN_ALIAS_RULES: ReadonlyArray<PatternAliasRule> = [
+  {
+    ruleID: "anthropic-thinking-suffix",
+    match: (normalizedModelID) => normalizedModelID.startsWith("claude-") && normalizedModelID.endsWith("-thinking"),
+    canonicalize: (normalizedModelID) => normalizedModelID.replace(/-thinking$/i, ""),
+  },
+]
+
+function normalizeLookupModelID(modelID: string): string {
+  return modelID.trim().toLowerCase()
+}
+
+export function resolveModelIDAlias(modelID: string): ModelIDAliasResolution {
+  const normalizedModelID = normalizeLookupModelID(modelID)
+  const exactRule = EXACT_ALIAS_RULES[normalizedModelID]
+  if (exactRule) {
+    return {
+      requestedModelID: normalizedModelID,
+      canonicalModelID: exactRule.canonicalModelID,
+      source: "exact-alias",
+      ruleID: exactRule.ruleID,
+    }
+  }
+
+  for (const rule of PATTERN_ALIAS_RULES) {
+    if (!rule.match(normalizedModelID)) {
+      continue
+    }
+
+    return {
+      requestedModelID: normalizedModelID,
+      canonicalModelID: rule.canonicalize(normalizedModelID),
+      source: "pattern-alias",
+      ruleID: rule.ruleID,
+    }
+  }
+
+  return {
+    requestedModelID: normalizedModelID,
+    canonicalModelID: normalizedModelID,
+    source: "canonical",
+  }
+}
@@ -24,14 +24,14 @@ export const HEURISTIC_MODEL_FAMILY_REGISTRY: ReadonlyArray<HeuristicModelFamily
  },
  {
    family: "openai-reasoning",
-    pattern: /^o\d(?:$|-)/,
+    pattern: /(?:^|\/)o\d(?:$|-)/,
    variants: ["low", "medium", "high"],
    reasoningEfforts: ["none", "minimal", "low", "medium", "high"],
  },
  {
    family: "gpt-5",
    includes: ["gpt-5"],
-    variants: ["low", "medium", "high", "xhigh", "max"],
+    variants: ["low", "medium", "high", "xhigh"],
    reasoningEfforts: ["none", "minimal", "low", "medium", "high", "xhigh"],
  },
  {
@@ -324,6 +324,27 @@ describe("resolveCompatibleModelSettings", () => {
    })
  })

+  test("GPT-5 downgrades unsupported max variant to xhigh", () => {
+    const result = resolveCompatibleModelSettings({
+      providerID: "openai",
+      modelID: "gpt-5.4",
+      desired: { variant: "max" },
+    })
+
+    expect(result).toEqual({
+      variant: "xhigh",
+      reasoningEffort: undefined,
+      changes: [
+        {
+          field: "variant",
+          from: "max",
+          to: "xhigh",
+          reason: "unsupported-by-model-family",
+        },
+      ],
+    })
+  })
+
  // Reasoning effort: "none" and "minimal" are valid per Vercel AI SDK
  test("GPT-5 keeps none reasoningEffort", () => {
    const result = resolveCompatibleModelSettings({