diff --git a/src/shared/model-capabilities.ts b/src/shared/model-capabilities.ts deleted file mode 100644 index 0a9749243..000000000 --- a/src/shared/model-capabilities.ts +++ /dev/null @@ -1,462 +0,0 @@ -import bundledModelCapabilitiesSnapshotJson from "../generated/model-capabilities.generated.json" -import { findProviderModelMetadata, type ModelMetadata } from "./connected-providers-cache" -import { resolveModelIDAlias } from "./model-capability-aliases" -import { detectHeuristicModelFamily } from "./model-capability-heuristics" - -export type ModelCapabilitiesSnapshotEntry = { - id: string - family?: string - reasoning?: boolean - temperature?: boolean - toolCall?: boolean - modalities?: { - input?: string[] - output?: string[] - } - limit?: { - context?: number - input?: number - output?: number - } -} - -export type ModelCapabilitiesSnapshot = { - generatedAt: string - sourceUrl: string - models: Record -} - -export type ModelCapabilities = { - requestedModelID: string - canonicalModelID: string - family?: string - variants?: string[] - reasoningEfforts?: string[] - reasoning?: boolean - supportsThinking?: boolean - supportsTemperature?: boolean - supportsTopP?: boolean - maxOutputTokens?: number - toolCall?: boolean - modalities?: { - input?: string[] - output?: string[] - } - diagnostics: ModelCapabilitiesDiagnostics -} - -type GetModelCapabilitiesInput = { - providerID: string - modelID: string - runtimeModel?: ModelMetadata | Record - runtimeSnapshot?: ModelCapabilitiesSnapshot - bundledSnapshot?: ModelCapabilitiesSnapshot -} - -type ModelCapabilityOverride = { - variants?: string[] - reasoningEfforts?: string[] - supportsThinking?: boolean - supportsTemperature?: boolean - supportsTopP?: boolean -} - -type DiagnosticSource = - | "none" - | "runtime" - | "runtime-snapshot" - | "bundled-snapshot" - | "override" - | "heuristic" - | "canonical" - | "exact-alias" - | "pattern-alias" - -export type ModelCapabilitiesDiagnostics = { - resolutionMode: "snapshot-backed" | "alias-backed" | "heuristic-backed" | "unknown" - canonicalization: { - source: "canonical" | "exact-alias" | "pattern-alias" - ruleID?: string - } - snapshot: { - source: "runtime-snapshot" | "bundled-snapshot" | "none" - } - family: { source: "snapshot" | "heuristic" | "none" } - variants: { source: Exclude } - reasoningEfforts: { source: Exclude } - reasoning: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" } - supportsThinking: { source: "runtime" | "override" | "heuristic" | "runtime-snapshot" | "bundled-snapshot" | "none" } - supportsTemperature: { source: "runtime" | "override" | "runtime-snapshot" | "bundled-snapshot" | "none" } - supportsTopP: { source: "runtime" | "override" | "none" } - maxOutputTokens: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" } - toolCall: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" } - modalities: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" } -} - -const MODEL_ID_OVERRIDES: Record = {} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value) -} - -function normalizeLookupModelID(modelID: string): string { - return modelID.trim().toLowerCase() -} - -function readBoolean(value: unknown): boolean | undefined { - return typeof value === "boolean" ? value : undefined -} - -function readNumber(value: unknown): number | undefined { - return typeof value === "number" ? value : undefined -} - -function readStringArray(value: unknown): string[] | undefined { - if (!Array.isArray(value)) { - return undefined - } - - const strings = value.filter((item): item is string => typeof item === "string") - return strings.length > 0 ? strings : undefined -} - -function normalizeVariantKeys(value: unknown): string[] | undefined { - const arrayVariants = readStringArray(value) - if (arrayVariants) { - return arrayVariants.map((variant) => variant.toLowerCase()) - } - - if (!isRecord(value)) { - return undefined - } - - const variants = Object.keys(value).map((variant) => variant.toLowerCase()) - return variants.length > 0 ? variants : undefined -} - -function readModalityKeys(value: unknown): string[] | undefined { - const stringArray = readStringArray(value) - if (stringArray) { - return stringArray.map((entry) => entry.toLowerCase()) - } - - if (!isRecord(value)) { - return undefined - } - - const enabled = Object.entries(value) - .filter(([, supported]) => supported === true) - .map(([modality]) => modality.toLowerCase()) - - return enabled.length > 0 ? enabled : undefined -} - -function normalizeModalities(value: unknown): ModelCapabilities["modalities"] | undefined { - if (!isRecord(value)) { - return undefined - } - - const input = readModalityKeys(value.input) - const output = readModalityKeys(value.output) - - if (!input && !output) { - return undefined - } - - return { - ...(input ? { input } : {}), - ...(output ? { output } : {}), - } -} - -function normalizeSnapshot(snapshot: ModelCapabilitiesSnapshot | typeof bundledModelCapabilitiesSnapshotJson): ModelCapabilitiesSnapshot { - return snapshot as ModelCapabilitiesSnapshot -} - -function getOverride(modelID: string): ModelCapabilityOverride | undefined { - return MODEL_ID_OVERRIDES[normalizeLookupModelID(modelID)] -} - -function readRuntimeModelCapabilities(runtimeModel: Record | undefined): Record | undefined { - return isRecord(runtimeModel?.capabilities) ? runtimeModel.capabilities : undefined -} - -function readRuntimeModelLimitOutput(runtimeModel: Record | undefined): number | undefined { - if (!runtimeModel) { - return undefined - } - - const limit = isRecord(runtimeModel.limit) - ? runtimeModel.limit - : readRuntimeModelCapabilities(runtimeModel)?.limit - if (!isRecord(limit)) { - return undefined - } - - return readNumber(limit.output) -} - -function readRuntimeModelBoolean(runtimeModel: Record | undefined, keys: string[]): boolean | undefined { - if (!runtimeModel) { - return undefined - } - - const runtimeCapabilities = readRuntimeModelCapabilities(runtimeModel) - - for (const key of keys) { - const value = runtimeModel[key] - if (typeof value === "boolean") { - return value - } - - const capabilityValue = runtimeCapabilities?.[key] - if (typeof capabilityValue === "boolean") { - return capabilityValue - } - } - - return undefined -} - -function readRuntimeModelModalities(runtimeModel: Record | undefined): ModelCapabilities["modalities"] | undefined { - if (!runtimeModel) { - return undefined - } - - const rootModalities = normalizeModalities(runtimeModel.modalities) - if (rootModalities) { - return rootModalities - } - - const runtimeCapabilities = readRuntimeModelCapabilities(runtimeModel) - if (!runtimeCapabilities) { - return undefined - } - - const nestedModalities = normalizeModalities(runtimeCapabilities.modalities) - if (nestedModalities) { - return nestedModalities - } - - const capabilityModalities = normalizeModalities(runtimeCapabilities) - if (capabilityModalities) { - return capabilityModalities - } - - return undefined -} - -function readRuntimeModelVariants(runtimeModel: Record | undefined): string[] | undefined { - if (!runtimeModel) { - return undefined - } - - const rootVariants = normalizeVariantKeys(runtimeModel.variants) - if (rootVariants) { - return rootVariants - } - - const runtimeCapabilities = readRuntimeModelCapabilities(runtimeModel) - if (!runtimeCapabilities) { - return undefined - } - - return normalizeVariantKeys(runtimeCapabilities.variants) -} - -function readRuntimeModelTopPSupport(runtimeModel: Record | undefined): boolean | undefined { - return readRuntimeModelBoolean(runtimeModel, ["topP", "top_p"]) -} - -function readRuntimeModelToolCallSupport(runtimeModel: Record | undefined): boolean | undefined { - return readRuntimeModelBoolean(runtimeModel, ["toolCall", "tool_call", "toolcall"]) -} - -function readRuntimeModelReasoningSupport(runtimeModel: Record | undefined): boolean | undefined { - return readRuntimeModelBoolean(runtimeModel, ["reasoning"]) -} - -function readRuntimeModelTemperatureSupport(runtimeModel: Record | undefined): boolean | undefined { - return readRuntimeModelBoolean(runtimeModel, ["temperature"]) -} - -function readRuntimeModelThinkingSupport(runtimeModel: Record | undefined): boolean | undefined { - const capabilityValue = readRuntimeModelReasoningSupport(runtimeModel) - if (capabilityValue !== undefined) { - return capabilityValue - } - - const rootThinkingSupport = readRuntimeModelBoolean(runtimeModel, ["thinking", "supportsThinking"]) - if (rootThinkingSupport !== undefined) { - return rootThinkingSupport - } - - const runtimeCapabilities = readRuntimeModelCapabilities(runtimeModel) - if (!runtimeCapabilities) { - return undefined - } - - for (const key of ["thinking", "supportsThinking"] as const) { - const value = runtimeCapabilities[key] - if (typeof value === "boolean") { - return value - } - } - - return undefined -} - -function readRuntimeModel(runtimeModel: ModelMetadata | Record | undefined): Record | undefined { - return isRecord(runtimeModel) ? runtimeModel : undefined -} - -const bundledModelCapabilitiesSnapshot = normalizeSnapshot(bundledModelCapabilitiesSnapshotJson) - -export function getBundledModelCapabilitiesSnapshot(): ModelCapabilitiesSnapshot { - return bundledModelCapabilitiesSnapshot -} - -export function getModelCapabilities(input: GetModelCapabilitiesInput): ModelCapabilities { - const canonicalization = resolveModelIDAlias(input.modelID) - const requestedModelID = canonicalization.requestedModelID - const canonicalModelID = canonicalization.canonicalModelID - const override = getOverride(input.modelID) - const runtimeModel = readRuntimeModel( - input.runtimeModel ?? findProviderModelMetadata(input.providerID, input.modelID), - ) - const runtimeSnapshot = input.runtimeSnapshot - const bundledSnapshot = input.bundledSnapshot ?? bundledModelCapabilitiesSnapshot - const snapshotEntry = runtimeSnapshot?.models?.[canonicalModelID] ?? bundledSnapshot.models[canonicalModelID] - const heuristicFamily = detectHeuristicModelFamily(canonicalModelID) - const runtimeVariants = readRuntimeModelVariants(runtimeModel) - const snapshotSource: ModelCapabilitiesDiagnostics["snapshot"]["source"] = - runtimeSnapshot?.models?.[canonicalModelID] - ? "runtime-snapshot" - : bundledSnapshot.models[canonicalModelID] - ? "bundled-snapshot" - : "none" - const familySource: ModelCapabilitiesDiagnostics["family"]["source"] = - snapshotEntry?.family - ? "snapshot" - : heuristicFamily?.family - ? "heuristic" - : "none" - const variantsSource: ModelCapabilitiesDiagnostics["variants"]["source"] = - runtimeVariants - ? "runtime" - : override?.variants - ? "override" - : heuristicFamily?.variants - ? "heuristic" - : "none" - const reasoningEffortsSource: ModelCapabilitiesDiagnostics["reasoningEfforts"]["source"] = - override?.reasoningEfforts - ? "override" - : heuristicFamily?.reasoningEfforts - ? "heuristic" - : "none" - const reasoningSource: ModelCapabilitiesDiagnostics["reasoning"]["source"] = - readRuntimeModelReasoningSupport(runtimeModel) !== undefined - ? "runtime" - : snapshotEntry?.reasoning !== undefined - ? snapshotSource - : "none" - const supportsThinkingSource: ModelCapabilitiesDiagnostics["supportsThinking"]["source"] = - override?.supportsThinking !== undefined - ? "override" - : heuristicFamily?.supportsThinking !== undefined - ? "heuristic" - : readRuntimeModelThinkingSupport(runtimeModel) !== undefined - ? "runtime" - : snapshotEntry?.reasoning !== undefined - ? snapshotSource - : "none" - const supportsTemperatureSource: ModelCapabilitiesDiagnostics["supportsTemperature"]["source"] = - readRuntimeModelTemperatureSupport(runtimeModel) !== undefined - ? "runtime" - : override?.supportsTemperature !== undefined - ? "override" - : snapshotEntry?.temperature !== undefined - ? snapshotSource - : "none" - const supportsTopPSource: ModelCapabilitiesDiagnostics["supportsTopP"]["source"] = - readRuntimeModelTopPSupport(runtimeModel) !== undefined - ? "runtime" - : override?.supportsTopP !== undefined - ? "override" - : "none" - const maxOutputTokensSource: ModelCapabilitiesDiagnostics["maxOutputTokens"]["source"] = - readRuntimeModelLimitOutput(runtimeModel) !== undefined - ? "runtime" - : snapshotEntry?.limit?.output !== undefined - ? snapshotSource - : "none" - const toolCallSource: ModelCapabilitiesDiagnostics["toolCall"]["source"] = - readRuntimeModelToolCallSupport(runtimeModel) !== undefined - ? "runtime" - : snapshotEntry?.toolCall !== undefined - ? snapshotSource - : "none" - const modalitiesSource: ModelCapabilitiesDiagnostics["modalities"]["source"] = - readRuntimeModelModalities(runtimeModel) !== undefined - ? "runtime" - : snapshotEntry?.modalities !== undefined - ? snapshotSource - : "none" - const resolutionMode: ModelCapabilitiesDiagnostics["resolutionMode"] = - snapshotSource !== "none" && canonicalization.source === "canonical" - ? "snapshot-backed" - : snapshotSource !== "none" - ? "alias-backed" - : familySource === "heuristic" || variantsSource === "heuristic" || reasoningEffortsSource === "heuristic" - ? "heuristic-backed" - : "unknown" - - return { - requestedModelID, - canonicalModelID, - family: snapshotEntry?.family ?? heuristicFamily?.family, - variants: runtimeVariants ?? override?.variants ?? heuristicFamily?.variants, - reasoningEfforts: override?.reasoningEfforts ?? heuristicFamily?.reasoningEfforts, - reasoning: readRuntimeModelReasoningSupport(runtimeModel) ?? snapshotEntry?.reasoning, - supportsThinking: - override?.supportsThinking - ?? heuristicFamily?.supportsThinking - ?? readRuntimeModelThinkingSupport(runtimeModel) - ?? snapshotEntry?.reasoning, - supportsTemperature: - readRuntimeModelTemperatureSupport(runtimeModel) - ?? override?.supportsTemperature - ?? snapshotEntry?.temperature, - supportsTopP: - readRuntimeModelTopPSupport(runtimeModel) - ?? override?.supportsTopP, - maxOutputTokens: - readRuntimeModelLimitOutput(runtimeModel) - ?? snapshotEntry?.limit?.output, - toolCall: - readRuntimeModelToolCallSupport(runtimeModel) - ?? snapshotEntry?.toolCall, - modalities: - readRuntimeModelModalities(runtimeModel) - ?? snapshotEntry?.modalities, - diagnostics: { - resolutionMode, - canonicalization: { - source: canonicalization.source, - ...(canonicalization.ruleID ? { ruleID: canonicalization.ruleID } : {}), - }, - snapshot: { source: snapshotSource }, - family: { source: familySource }, - variants: { source: variantsSource }, - reasoningEfforts: { source: reasoningEffortsSource }, - reasoning: { source: reasoningSource }, - supportsThinking: { source: supportsThinkingSource }, - supportsTemperature: { source: supportsTemperatureSource }, - supportsTopP: { source: supportsTopPSource }, - maxOutputTokens: { source: maxOutputTokensSource }, - toolCall: { source: toolCallSource }, - modalities: { source: modalitiesSource }, - }, - } -} diff --git a/src/shared/model-capabilities/bundled-snapshot.ts b/src/shared/model-capabilities/bundled-snapshot.ts new file mode 100644 index 000000000..65644a8cf --- /dev/null +++ b/src/shared/model-capabilities/bundled-snapshot.ts @@ -0,0 +1,15 @@ +import bundledModelCapabilitiesSnapshotJson from "../../generated/model-capabilities.generated.json" + +import type { ModelCapabilitiesSnapshot } from "./types" + +function normalizeSnapshot( + snapshot: ModelCapabilitiesSnapshot | typeof bundledModelCapabilitiesSnapshotJson, +): ModelCapabilitiesSnapshot { + return snapshot as ModelCapabilitiesSnapshot +} + +const bundledModelCapabilitiesSnapshot = normalizeSnapshot(bundledModelCapabilitiesSnapshotJson) + +export function getBundledModelCapabilitiesSnapshot(): ModelCapabilitiesSnapshot { + return bundledModelCapabilitiesSnapshot +} diff --git a/src/shared/model-capabilities/get-model-capabilities.ts b/src/shared/model-capabilities/get-model-capabilities.ts new file mode 100644 index 000000000..fa27f1e86 --- /dev/null +++ b/src/shared/model-capabilities/get-model-capabilities.ts @@ -0,0 +1,140 @@ +import { findProviderModelMetadata } from "../connected-providers-cache" +import { resolveModelIDAlias } from "../model-capability-aliases" +import { detectHeuristicModelFamily } from "../model-capability-heuristics" + +import { getBundledModelCapabilitiesSnapshot } from "./bundled-snapshot" +import { + readRuntimeModel, + readRuntimeModelLimitOutput, + readRuntimeModelModalities, + readRuntimeModelReasoningSupport, + readRuntimeModelTemperatureSupport, + readRuntimeModelThinkingSupport, + readRuntimeModelToolCallSupport, + readRuntimeModelTopPSupport, + readRuntimeModelVariants, +} from "./runtime-model-readers" +import type { + GetModelCapabilitiesInput, + ModelCapabilities, + ModelCapabilitiesDiagnostics, + ModelCapabilityOverride, +} from "./types" + +const MODEL_ID_OVERRIDES: Record = {} + +function normalizeLookupModelID(modelID: string): string { + return modelID.trim().toLowerCase() +} + +function getOverride(modelID: string): ModelCapabilityOverride | undefined { + return MODEL_ID_OVERRIDES[normalizeLookupModelID(modelID)] +} + +export function getModelCapabilities(input: GetModelCapabilitiesInput): ModelCapabilities { + const canonicalization = resolveModelIDAlias(input.modelID) + const override = getOverride(input.modelID) + const runtimeModel = readRuntimeModel( + input.runtimeModel ?? findProviderModelMetadata(input.providerID, input.modelID), + ) + const runtimeSnapshot = input.runtimeSnapshot + const bundledSnapshot = input.bundledSnapshot ?? getBundledModelCapabilitiesSnapshot() + const snapshotEntry = runtimeSnapshot?.models?.[canonicalization.canonicalModelID] + ?? bundledSnapshot.models[canonicalization.canonicalModelID] + const heuristicFamily = detectHeuristicModelFamily(canonicalization.canonicalModelID) + + const runtimeVariants = readRuntimeModelVariants(runtimeModel) + const runtimeReasoning = readRuntimeModelReasoningSupport(runtimeModel) + const runtimeThinking = readRuntimeModelThinkingSupport(runtimeModel) + const runtimeTemperature = readRuntimeModelTemperatureSupport(runtimeModel) + const runtimeTopP = readRuntimeModelTopPSupport(runtimeModel) + const runtimeMaxOutputTokens = readRuntimeModelLimitOutput(runtimeModel) + const runtimeToolCall = readRuntimeModelToolCallSupport(runtimeModel) + const runtimeModalities = readRuntimeModelModalities(runtimeModel) + + const snapshotSource: ModelCapabilitiesDiagnostics["snapshot"]["source"] = + runtimeSnapshot?.models?.[canonicalization.canonicalModelID] + ? "runtime-snapshot" + : bundledSnapshot.models[canonicalization.canonicalModelID] + ? "bundled-snapshot" + : "none" + const familySource: ModelCapabilitiesDiagnostics["family"]["source"] = + snapshotEntry?.family ? "snapshot" : heuristicFamily?.family ? "heuristic" : "none" + const variantsSource: ModelCapabilitiesDiagnostics["variants"]["source"] = + runtimeVariants ? "runtime" : override?.variants ? "override" : heuristicFamily?.variants ? "heuristic" : "none" + const reasoningEffortsSource: ModelCapabilitiesDiagnostics["reasoningEfforts"]["source"] = + override?.reasoningEfforts ? "override" : heuristicFamily?.reasoningEfforts ? "heuristic" : "none" + const reasoningSource: ModelCapabilitiesDiagnostics["reasoning"]["source"] = + runtimeReasoning === undefined ? snapshotEntry?.reasoning === undefined ? "none" : snapshotSource : "runtime" + const supportsThinkingSource: ModelCapabilitiesDiagnostics["supportsThinking"]["source"] = + override?.supportsThinking !== undefined + ? "override" + : heuristicFamily?.supportsThinking !== undefined + ? "heuristic" + : runtimeThinking !== undefined + ? "runtime" + : snapshotEntry?.reasoning !== undefined + ? snapshotSource + : "none" + const supportsTemperatureSource: ModelCapabilitiesDiagnostics["supportsTemperature"]["source"] = + runtimeTemperature !== undefined + ? "runtime" + : override?.supportsTemperature !== undefined + ? "override" + : snapshotEntry?.temperature !== undefined + ? snapshotSource + : "none" + const supportsTopPSource: ModelCapabilitiesDiagnostics["supportsTopP"]["source"] = + runtimeTopP !== undefined ? "runtime" : override?.supportsTopP !== undefined ? "override" : "none" + const maxOutputTokensSource: ModelCapabilitiesDiagnostics["maxOutputTokens"]["source"] = + runtimeMaxOutputTokens !== undefined + ? "runtime" + : snapshotEntry?.limit?.output !== undefined + ? snapshotSource + : "none" + const toolCallSource: ModelCapabilitiesDiagnostics["toolCall"]["source"] = + runtimeToolCall !== undefined ? "runtime" : snapshotEntry?.toolCall !== undefined ? snapshotSource : "none" + const modalitiesSource: ModelCapabilitiesDiagnostics["modalities"]["source"] = + runtimeModalities !== undefined ? "runtime" : snapshotEntry?.modalities !== undefined ? snapshotSource : "none" + const resolutionMode: ModelCapabilitiesDiagnostics["resolutionMode"] = + snapshotSource !== "none" && canonicalization.source === "canonical" + ? "snapshot-backed" + : snapshotSource !== "none" + ? "alias-backed" + : familySource === "heuristic" || variantsSource === "heuristic" || reasoningEffortsSource === "heuristic" + ? "heuristic-backed" + : "unknown" + + return { + requestedModelID: canonicalization.requestedModelID, + canonicalModelID: canonicalization.canonicalModelID, + family: snapshotEntry?.family ?? heuristicFamily?.family, + variants: runtimeVariants ?? override?.variants ?? heuristicFamily?.variants, + reasoningEfforts: override?.reasoningEfforts ?? heuristicFamily?.reasoningEfforts, + reasoning: runtimeReasoning ?? snapshotEntry?.reasoning, + supportsThinking: override?.supportsThinking ?? heuristicFamily?.supportsThinking ?? runtimeThinking ?? snapshotEntry?.reasoning, + supportsTemperature: runtimeTemperature ?? override?.supportsTemperature ?? snapshotEntry?.temperature, + supportsTopP: runtimeTopP ?? override?.supportsTopP, + maxOutputTokens: runtimeMaxOutputTokens ?? snapshotEntry?.limit?.output, + toolCall: runtimeToolCall ?? snapshotEntry?.toolCall, + modalities: runtimeModalities ?? snapshotEntry?.modalities, + diagnostics: { + resolutionMode, + canonicalization: { + source: canonicalization.source, + ...(canonicalization.ruleID ? { ruleID: canonicalization.ruleID } : {}), + }, + snapshot: { source: snapshotSource }, + family: { source: familySource }, + variants: { source: variantsSource }, + reasoningEfforts: { source: reasoningEffortsSource }, + reasoning: { source: reasoningSource }, + supportsThinking: { source: supportsThinkingSource }, + supportsTemperature: { source: supportsTemperatureSource }, + supportsTopP: { source: supportsTopPSource }, + maxOutputTokens: { source: maxOutputTokensSource }, + toolCall: { source: toolCallSource }, + modalities: { source: modalitiesSource }, + }, + } +} diff --git a/src/shared/model-capabilities/index.ts b/src/shared/model-capabilities/index.ts new file mode 100644 index 000000000..99549195a --- /dev/null +++ b/src/shared/model-capabilities/index.ts @@ -0,0 +1,9 @@ +export { getBundledModelCapabilitiesSnapshot } from "./bundled-snapshot" +export { getModelCapabilities } from "./get-model-capabilities" +export type { + GetModelCapabilitiesInput, + ModelCapabilities, + ModelCapabilitiesDiagnostics, + ModelCapabilitiesSnapshot, + ModelCapabilitiesSnapshotEntry, +} from "./types" diff --git a/src/shared/model-capabilities/runtime-model-readers.ts b/src/shared/model-capabilities/runtime-model-readers.ts new file mode 100644 index 000000000..a7b740f32 --- /dev/null +++ b/src/shared/model-capabilities/runtime-model-readers.ts @@ -0,0 +1,190 @@ +import type { ModelMetadata } from "../connected-providers-cache" + +import type { ModelCapabilities } from "./types" + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +function readNumber(value: unknown): number | undefined { + return typeof value === "number" ? value : undefined +} + +function readStringArray(value: unknown): string[] | undefined { + if (!Array.isArray(value)) { + return undefined + } + + const strings = value.filter((item): item is string => typeof item === "string") + return strings.length > 0 ? strings : undefined +} + +function normalizeVariantKeys(value: unknown): string[] | undefined { + const arrayVariants = readStringArray(value) + if (arrayVariants) { + return arrayVariants.map((variant) => variant.toLowerCase()) + } + + if (!isRecord(value)) { + return undefined + } + + const variants = Object.keys(value).map((variant) => variant.toLowerCase()) + return variants.length > 0 ? variants : undefined +} + +function readModalityKeys(value: unknown): string[] | undefined { + const stringArray = readStringArray(value) + if (stringArray) { + return stringArray.map((entry) => entry.toLowerCase()) + } + + if (!isRecord(value)) { + return undefined + } + + const enabled = Object.entries(value) + .filter(([, supported]) => supported === true) + .map(([modality]) => modality.toLowerCase()) + + return enabled.length > 0 ? enabled : undefined +} + +function normalizeModalities(value: unknown): ModelCapabilities["modalities"] | undefined { + if (!isRecord(value)) { + return undefined + } + + const input = readModalityKeys(value.input) + const output = readModalityKeys(value.output) + + if (!input && !output) { + return undefined + } + + return { + ...(input ? { input } : {}), + ...(output ? { output } : {}), + } +} + +function readRuntimeModelCapabilities( + runtimeModel: Record | undefined, +): Record | undefined { + return isRecord(runtimeModel?.capabilities) ? runtimeModel.capabilities : undefined +} + +function readRuntimeModelBoolean( + runtimeModel: Record | undefined, + keys: string[], +): boolean | undefined { + const runtimeCapabilities = readRuntimeModelCapabilities(runtimeModel) + + for (const key of keys) { + const value = runtimeModel?.[key] + if (typeof value === "boolean") { + return value + } + + const capabilityValue = runtimeCapabilities?.[key] + if (typeof capabilityValue === "boolean") { + return capabilityValue + } + } + + return undefined +} + +export function readRuntimeModel( + runtimeModel: ModelMetadata | Record | undefined, +): Record | undefined { + return isRecord(runtimeModel) ? runtimeModel : undefined +} + +export function readRuntimeModelVariants( + runtimeModel: Record | undefined, +): string[] | undefined { + const rootVariants = normalizeVariantKeys(runtimeModel?.variants) + if (rootVariants) { + return rootVariants + } + + return normalizeVariantKeys(readRuntimeModelCapabilities(runtimeModel)?.variants) +} + +export function readRuntimeModelModalities( + runtimeModel: Record | undefined, +): ModelCapabilities["modalities"] | undefined { + const rootModalities = normalizeModalities(runtimeModel?.modalities) + if (rootModalities) { + return rootModalities + } + + const runtimeCapabilities = readRuntimeModelCapabilities(runtimeModel) + return ( + normalizeModalities(runtimeCapabilities?.modalities) + ?? normalizeModalities(runtimeCapabilities) + ) +} + +export function readRuntimeModelReasoningSupport( + runtimeModel: Record | undefined, +): boolean | undefined { + return readRuntimeModelBoolean(runtimeModel, ["reasoning"]) +} + +export function readRuntimeModelThinkingSupport( + runtimeModel: Record | undefined, +): boolean | undefined { + const capabilityValue = readRuntimeModelReasoningSupport(runtimeModel) + if (capabilityValue !== undefined) { + return capabilityValue + } + + const thinkingSupport = readRuntimeModelBoolean(runtimeModel, ["thinking", "supportsThinking"]) + if (thinkingSupport !== undefined) { + return thinkingSupport + } + + const runtimeCapabilities = readRuntimeModelCapabilities(runtimeModel) + for (const key of ["thinking", "supportsThinking"] as const) { + const value = runtimeCapabilities?.[key] + if (typeof value === "boolean") { + return value + } + } + + return undefined +} + +export function readRuntimeModelTemperatureSupport( + runtimeModel: Record | undefined, +): boolean | undefined { + return readRuntimeModelBoolean(runtimeModel, ["temperature"]) +} + +export function readRuntimeModelTopPSupport( + runtimeModel: Record | undefined, +): boolean | undefined { + return readRuntimeModelBoolean(runtimeModel, ["topP", "top_p"]) +} + +export function readRuntimeModelToolCallSupport( + runtimeModel: Record | undefined, +): boolean | undefined { + return readRuntimeModelBoolean(runtimeModel, ["toolCall", "tool_call", "toolcall"]) +} + +export function readRuntimeModelLimitOutput( + runtimeModel: Record | undefined, +): number | undefined { + const limit = isRecord(runtimeModel?.limit) + ? runtimeModel.limit + : readRuntimeModelCapabilities(runtimeModel)?.limit + + if (!isRecord(limit)) { + return undefined + } + + return readNumber(limit.output) +} diff --git a/src/shared/model-capabilities/types.ts b/src/shared/model-capabilities/types.ts new file mode 100644 index 000000000..74881c72e --- /dev/null +++ b/src/shared/model-capabilities/types.ts @@ -0,0 +1,80 @@ +import type { ModelMetadata } from "../connected-providers-cache" + +export type ModelCapabilitiesSnapshotEntry = { + id: string + family?: string + reasoning?: boolean + temperature?: boolean + toolCall?: boolean + modalities?: { + input?: string[] + output?: string[] + } + limit?: { + context?: number + input?: number + output?: number + } +} + +export type ModelCapabilitiesSnapshot = { + generatedAt: string + sourceUrl: string + models: Record +} + +export type ModelCapabilitiesDiagnostics = { + resolutionMode: "snapshot-backed" | "alias-backed" | "heuristic-backed" | "unknown" + canonicalization: { + source: "canonical" | "exact-alias" | "pattern-alias" + ruleID?: string + } + snapshot: { + source: "runtime-snapshot" | "bundled-snapshot" | "none" + } + family: { source: "snapshot" | "heuristic" | "none" } + variants: { source: "none" | "runtime" | "override" | "heuristic" | "canonical" } + reasoningEfforts: { source: "none" | "override" | "heuristic" } + reasoning: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" } + supportsThinking: { source: "runtime" | "override" | "heuristic" | "runtime-snapshot" | "bundled-snapshot" | "none" } + supportsTemperature: { source: "runtime" | "override" | "runtime-snapshot" | "bundled-snapshot" | "none" } + supportsTopP: { source: "runtime" | "override" | "none" } + maxOutputTokens: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" } + toolCall: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" } + modalities: { source: "runtime" | "runtime-snapshot" | "bundled-snapshot" | "none" } +} + +export type ModelCapabilities = { + requestedModelID: string + canonicalModelID: string + family?: string + variants?: string[] + reasoningEfforts?: string[] + reasoning?: boolean + supportsThinking?: boolean + supportsTemperature?: boolean + supportsTopP?: boolean + maxOutputTokens?: number + toolCall?: boolean + modalities?: { + input?: string[] + output?: string[] + } + diagnostics: ModelCapabilitiesDiagnostics +} + +export type GetModelCapabilitiesInput = { + providerID: string + modelID: string + runtimeModel?: ModelMetadata | Record + runtimeSnapshot?: ModelCapabilitiesSnapshot + bundledSnapshot?: ModelCapabilitiesSnapshot +} + +export type ModelCapabilityOverride = { + variants?: string[] + reasoningEfforts?: string[] + supportsThinking?: boolean + supportsTemperature?: boolean + supportsTopP?: boolean +}