test(fallback): rename quotio matrix to cliproxyapi
This commit is contained in:
+53
-91
@@ -1,10 +1,12 @@
|
|||||||
declare const require: (name: string) => any
|
declare const require: (name: string) => any
|
||||||
const { afterEach, describe, expect, mock, test } = require("bun:test")
|
const { afterEach, describe, expect, mock, test } = require("bun:test")
|
||||||
|
|
||||||
|
const PROVIDER_ID = "cliproxyapi"
|
||||||
|
|
||||||
mock.module("../shared/connected-providers-cache", () => ({
|
mock.module("../shared/connected-providers-cache", () => ({
|
||||||
readConnectedProvidersCache: () => ["quotio"],
|
readConnectedProvidersCache: () => [PROVIDER_ID],
|
||||||
readProviderModelsCache: () => ({
|
readProviderModelsCache: () => ({
|
||||||
connected: ["quotio"],
|
connected: [PROVIDER_ID],
|
||||||
}),
|
}),
|
||||||
}))
|
}))
|
||||||
|
|
||||||
@@ -16,16 +18,21 @@ import { _resetForTesting } from "../features/claude-code-session-state"
|
|||||||
import { SessionCategoryRegistry } from "../shared/session-category-registry"
|
import { SessionCategoryRegistry } from "../shared/session-category-registry"
|
||||||
|
|
||||||
const PRIMARY_MODEL = {
|
const PRIMARY_MODEL = {
|
||||||
providerID: "quotio",
|
providerID: PROVIDER_ID,
|
||||||
modelID: "claude-opus-4-6",
|
modelID: "claude-opus-4-6",
|
||||||
}
|
}
|
||||||
|
|
||||||
const PRIMARY_MODEL_STRING = `${PRIMARY_MODEL.providerID}/${PRIMARY_MODEL.modelID}`
|
const PRIMARY_MODEL_STRING = `${PRIMARY_MODEL.providerID}/${PRIMARY_MODEL.modelID}`
|
||||||
|
|
||||||
const QUOTIO_FALLBACKS = [
|
const FIRST_FALLBACK_MODEL = {
|
||||||
"quotio/claude-sonnet-4-6",
|
providerID: PROVIDER_ID,
|
||||||
"quotio/gpt-5.4",
|
modelID: "claude-sonnet-4-6",
|
||||||
"quotio/kimi-k2.5",
|
}
|
||||||
|
|
||||||
|
const CLIPROXYAPI_FALLBACKS = [
|
||||||
|
`${PROVIDER_ID}/claude-sonnet-4-6`,
|
||||||
|
`${PROVIDER_ID}/gpt-5.4`,
|
||||||
|
`${PROVIDER_ID}/kimi-k2.5`,
|
||||||
]
|
]
|
||||||
|
|
||||||
type HarnessMode = "none" | "model" | "runtime" | "both"
|
type HarnessMode = "none" | "model" | "runtime" | "both"
|
||||||
@@ -41,7 +48,7 @@ function createPluginConfig(mode: HarnessMode) {
|
|||||||
return {
|
return {
|
||||||
agents: {
|
agents: {
|
||||||
sisyphus: {
|
sisyphus: {
|
||||||
fallback_models: QUOTIO_FALLBACKS,
|
fallback_models: CLIPROXYAPI_FALLBACKS,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
...(mode === "runtime" || mode === "both"
|
...(mode === "runtime" || mode === "both"
|
||||||
@@ -244,7 +251,7 @@ async function triggerSessionError(
|
|||||||
model: PRIMARY_MODEL_STRING,
|
model: PRIMARY_MODEL_STRING,
|
||||||
error: {
|
error: {
|
||||||
statusCode: 529,
|
statusCode: 529,
|
||||||
message: "Overloaded upstream for quotio/claude-opus-4-6",
|
message: `Overloaded upstream for ${PRIMARY_MODEL_STRING}`,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -294,7 +301,7 @@ async function triggerAssistantMessageError(
|
|||||||
path: { cwd: "/tmp", root: "/tmp" },
|
path: { cwd: "/tmp", root: "/tmp" },
|
||||||
error: {
|
error: {
|
||||||
statusCode: 529,
|
statusCode: 529,
|
||||||
message: "Overloaded upstream for quotio/claude-opus-4-6",
|
message: `Overloaded upstream for ${PRIMARY_MODEL_STRING}`,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -307,9 +314,9 @@ afterEach(() => {
|
|||||||
SessionCategoryRegistry.clear()
|
SessionCategoryRegistry.clear()
|
||||||
})
|
})
|
||||||
|
|
||||||
describe("Quotio-only fallback matrix", () => {
|
describe("CLIProxyAPI-only fallback matrix", () => {
|
||||||
test("no fallback leaves retryable session.error on the primary Quotio model", async () => {
|
test("no fallback leaves retryable session.error on the primary CLIProxyAPI model", async () => {
|
||||||
const sessionID = "quotio-none-session-error"
|
const sessionID = "cliproxyapi-none-session-error"
|
||||||
const harness = createHarness({ mode: "none" })
|
const harness = createHarness({ mode: "none" })
|
||||||
|
|
||||||
await primeMainSession(harness.eventHandler, sessionID)
|
await primeMainSession(harness.eventHandler, sessionID)
|
||||||
@@ -327,8 +334,8 @@ describe("Quotio-only fallback matrix", () => {
|
|||||||
expect(output.message["model"]).toBeUndefined()
|
expect(output.message["model"]).toBeUndefined()
|
||||||
})
|
})
|
||||||
|
|
||||||
test("model fallback switches Quotio session.error failures to the next Quotio model", async () => {
|
test("model fallback switches CLIProxyAPI session.error failures to the next CLIProxyAPI model", async () => {
|
||||||
const sessionID = "quotio-model-session-error"
|
const sessionID = "cliproxyapi-model-session-error"
|
||||||
const harness = createHarness({ mode: "model" })
|
const harness = createHarness({ mode: "model" })
|
||||||
|
|
||||||
await primeMainSession(harness.eventHandler, sessionID)
|
await primeMainSession(harness.eventHandler, sessionID)
|
||||||
@@ -343,14 +350,11 @@ describe("Quotio-only fallback matrix", () => {
|
|||||||
expect(harness.abortCalls).toEqual([sessionID])
|
expect(harness.abortCalls).toEqual([sessionID])
|
||||||
expect(harness.promptCalls).toEqual([sessionID])
|
expect(harness.promptCalls).toEqual([sessionID])
|
||||||
expect(harness.promptAsyncCalls).toEqual([])
|
expect(harness.promptAsyncCalls).toEqual([])
|
||||||
expect(output.message["model"]).toEqual({
|
expect(output.message["model"]).toEqual(FIRST_FALLBACK_MODEL)
|
||||||
providerID: "quotio",
|
|
||||||
modelID: "claude-sonnet-4-6",
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
|
||||||
test("model fallback switches Quotio session.status retry signals to the next Quotio model", async () => {
|
test("model fallback switches CLIProxyAPI session.status retry signals to the next CLIProxyAPI model", async () => {
|
||||||
const sessionID = "quotio-model-session-status"
|
const sessionID = "cliproxyapi-model-session-status"
|
||||||
const harness = createHarness({ mode: "model" })
|
const harness = createHarness({ mode: "model" })
|
||||||
|
|
||||||
await primeMainSession(harness.eventHandler, sessionID)
|
await primeMainSession(harness.eventHandler, sessionID)
|
||||||
@@ -365,14 +369,11 @@ describe("Quotio-only fallback matrix", () => {
|
|||||||
expect(harness.abortCalls).toEqual([sessionID])
|
expect(harness.abortCalls).toEqual([sessionID])
|
||||||
expect(harness.promptCalls).toEqual([sessionID])
|
expect(harness.promptCalls).toEqual([sessionID])
|
||||||
expect(harness.promptAsyncCalls).toEqual([])
|
expect(harness.promptAsyncCalls).toEqual([])
|
||||||
expect(output.message["model"]).toEqual({
|
expect(output.message["model"]).toEqual(FIRST_FALLBACK_MODEL)
|
||||||
providerID: "quotio",
|
|
||||||
modelID: "claude-sonnet-4-6",
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
|
||||||
test("model fallback switches Quotio assistant message.updated errors to the next Quotio model", async () => {
|
test("model fallback switches CLIProxyAPI assistant message.updated errors to the next CLIProxyAPI model", async () => {
|
||||||
const sessionID = "quotio-model-message-updated"
|
const sessionID = "cliproxyapi-model-message-updated"
|
||||||
const harness = createHarness({ mode: "model" })
|
const harness = createHarness({ mode: "model" })
|
||||||
|
|
||||||
await primeMainSession(harness.eventHandler, sessionID)
|
await primeMainSession(harness.eventHandler, sessionID)
|
||||||
@@ -387,14 +388,11 @@ describe("Quotio-only fallback matrix", () => {
|
|||||||
expect(harness.abortCalls).toEqual([sessionID])
|
expect(harness.abortCalls).toEqual([sessionID])
|
||||||
expect(harness.promptCalls).toEqual([sessionID])
|
expect(harness.promptCalls).toEqual([sessionID])
|
||||||
expect(harness.promptAsyncCalls).toEqual([])
|
expect(harness.promptAsyncCalls).toEqual([])
|
||||||
expect(output.message["model"]).toEqual({
|
expect(output.message["model"]).toEqual(FIRST_FALLBACK_MODEL)
|
||||||
providerID: "quotio",
|
|
||||||
modelID: "claude-sonnet-4-6",
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
|
||||||
test("runtime fallback retries Quotio session.error failures through promptAsync and overrides the next message model", async () => {
|
test("runtime fallback retries CLIProxyAPI session.error failures through promptAsync and overrides the next message model", async () => {
|
||||||
const sessionID = "quotio-runtime-session-error"
|
const sessionID = "cliproxyapi-runtime-session-error"
|
||||||
const harness = createHarness({ mode: "runtime" })
|
const harness = createHarness({ mode: "runtime" })
|
||||||
|
|
||||||
await primeMainSession(harness.eventHandler, sessionID)
|
await primeMainSession(harness.eventHandler, sessionID)
|
||||||
@@ -408,18 +406,12 @@ describe("Quotio-only fallback matrix", () => {
|
|||||||
expect(harness.abortCalls).toEqual([])
|
expect(harness.abortCalls).toEqual([])
|
||||||
expect(harness.promptCalls).toEqual([])
|
expect(harness.promptCalls).toEqual([])
|
||||||
expect(harness.promptAsyncCalls).toHaveLength(1)
|
expect(harness.promptAsyncCalls).toHaveLength(1)
|
||||||
expect(harness.promptAsyncCalls[0]?.model).toEqual({
|
expect(harness.promptAsyncCalls[0]?.model).toEqual(FIRST_FALLBACK_MODEL)
|
||||||
providerID: "quotio",
|
expect(output.message["model"]).toEqual(FIRST_FALLBACK_MODEL)
|
||||||
modelID: "claude-sonnet-4-6",
|
|
||||||
})
|
|
||||||
expect(output.message["model"]).toEqual({
|
|
||||||
providerID: "quotio",
|
|
||||||
modelID: "claude-sonnet-4-6",
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
|
||||||
test("runtime fallback retries Quotio session.status auto-retry signals through promptAsync", async () => {
|
test("runtime fallback retries CLIProxyAPI session.status auto-retry signals through promptAsync", async () => {
|
||||||
const sessionID = "quotio-runtime-session-status"
|
const sessionID = "cliproxyapi-runtime-session-status"
|
||||||
const harness = createHarness({ mode: "runtime" })
|
const harness = createHarness({ mode: "runtime" })
|
||||||
|
|
||||||
await primeMainSession(harness.eventHandler, sessionID)
|
await primeMainSession(harness.eventHandler, sessionID)
|
||||||
@@ -433,18 +425,12 @@ describe("Quotio-only fallback matrix", () => {
|
|||||||
expect(harness.abortCalls).toEqual([sessionID])
|
expect(harness.abortCalls).toEqual([sessionID])
|
||||||
expect(harness.promptCalls).toEqual([])
|
expect(harness.promptCalls).toEqual([])
|
||||||
expect(harness.promptAsyncCalls).toHaveLength(1)
|
expect(harness.promptAsyncCalls).toHaveLength(1)
|
||||||
expect(harness.promptAsyncCalls[0]?.model).toEqual({
|
expect(harness.promptAsyncCalls[0]?.model).toEqual(FIRST_FALLBACK_MODEL)
|
||||||
providerID: "quotio",
|
expect(output.message["model"]).toEqual(FIRST_FALLBACK_MODEL)
|
||||||
modelID: "claude-sonnet-4-6",
|
|
||||||
})
|
|
||||||
expect(output.message["model"]).toEqual({
|
|
||||||
providerID: "quotio",
|
|
||||||
modelID: "claude-sonnet-4-6",
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
|
||||||
test("runtime fallback retries Quotio assistant message.updated errors through promptAsync", async () => {
|
test("runtime fallback retries CLIProxyAPI assistant message.updated errors through promptAsync", async () => {
|
||||||
const sessionID = "quotio-runtime-message-updated"
|
const sessionID = "cliproxyapi-runtime-message-updated"
|
||||||
const harness = createHarness({ mode: "runtime" })
|
const harness = createHarness({ mode: "runtime" })
|
||||||
|
|
||||||
await primeMainSession(harness.eventHandler, sessionID)
|
await primeMainSession(harness.eventHandler, sessionID)
|
||||||
@@ -458,18 +444,12 @@ describe("Quotio-only fallback matrix", () => {
|
|||||||
expect(harness.abortCalls).toEqual([])
|
expect(harness.abortCalls).toEqual([])
|
||||||
expect(harness.promptCalls).toEqual([])
|
expect(harness.promptCalls).toEqual([])
|
||||||
expect(harness.promptAsyncCalls).toHaveLength(1)
|
expect(harness.promptAsyncCalls).toHaveLength(1)
|
||||||
expect(harness.promptAsyncCalls[0]?.model).toEqual({
|
expect(harness.promptAsyncCalls[0]?.model).toEqual(FIRST_FALLBACK_MODEL)
|
||||||
providerID: "quotio",
|
expect(output.message["model"]).toEqual(FIRST_FALLBACK_MODEL)
|
||||||
modelID: "claude-sonnet-4-6",
|
|
||||||
})
|
|
||||||
expect(output.message["model"]).toEqual({
|
|
||||||
providerID: "quotio",
|
|
||||||
modelID: "claude-sonnet-4-6",
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
|
||||||
test("model+runtime prefers the runtime path for Quotio session.error failures", async () => {
|
test("model+runtime prefers the runtime path for CLIProxyAPI session.error failures", async () => {
|
||||||
const sessionID = "quotio-both-session-error"
|
const sessionID = "cliproxyapi-both-session-error"
|
||||||
const harness = createHarness({ mode: "both" })
|
const harness = createHarness({ mode: "both" })
|
||||||
|
|
||||||
await primeMainSession(harness.eventHandler, sessionID)
|
await primeMainSession(harness.eventHandler, sessionID)
|
||||||
@@ -483,18 +463,12 @@ describe("Quotio-only fallback matrix", () => {
|
|||||||
expect(harness.abortCalls).toEqual([])
|
expect(harness.abortCalls).toEqual([])
|
||||||
expect(harness.promptCalls).toEqual([])
|
expect(harness.promptCalls).toEqual([])
|
||||||
expect(harness.promptAsyncCalls).toHaveLength(1)
|
expect(harness.promptAsyncCalls).toHaveLength(1)
|
||||||
expect(harness.promptAsyncCalls[0]?.model).toEqual({
|
expect(harness.promptAsyncCalls[0]?.model).toEqual(FIRST_FALLBACK_MODEL)
|
||||||
providerID: "quotio",
|
expect(output.message["model"]).toEqual(FIRST_FALLBACK_MODEL)
|
||||||
modelID: "claude-sonnet-4-6",
|
|
||||||
})
|
|
||||||
expect(output.message["model"]).toEqual({
|
|
||||||
providerID: "quotio",
|
|
||||||
modelID: "claude-sonnet-4-6",
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
|
||||||
test("model+runtime prefers the runtime path for Quotio session.status retry signals", async () => {
|
test("model+runtime prefers the runtime path for CLIProxyAPI session.status retry signals", async () => {
|
||||||
const sessionID = "quotio-both-session-status"
|
const sessionID = "cliproxyapi-both-session-status"
|
||||||
const harness = createHarness({ mode: "both" })
|
const harness = createHarness({ mode: "both" })
|
||||||
|
|
||||||
await primeMainSession(harness.eventHandler, sessionID)
|
await primeMainSession(harness.eventHandler, sessionID)
|
||||||
@@ -508,18 +482,12 @@ describe("Quotio-only fallback matrix", () => {
|
|||||||
expect(harness.abortCalls).toEqual([sessionID])
|
expect(harness.abortCalls).toEqual([sessionID])
|
||||||
expect(harness.promptCalls).toEqual([])
|
expect(harness.promptCalls).toEqual([])
|
||||||
expect(harness.promptAsyncCalls).toHaveLength(1)
|
expect(harness.promptAsyncCalls).toHaveLength(1)
|
||||||
expect(harness.promptAsyncCalls[0]?.model).toEqual({
|
expect(harness.promptAsyncCalls[0]?.model).toEqual(FIRST_FALLBACK_MODEL)
|
||||||
providerID: "quotio",
|
expect(output.message["model"]).toEqual(FIRST_FALLBACK_MODEL)
|
||||||
modelID: "claude-sonnet-4-6",
|
|
||||||
})
|
|
||||||
expect(output.message["model"]).toEqual({
|
|
||||||
providerID: "quotio",
|
|
||||||
modelID: "claude-sonnet-4-6",
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
|
||||||
test("model+runtime prefers the runtime path for Quotio assistant message.updated errors", async () => {
|
test("model+runtime prefers the runtime path for CLIProxyAPI assistant message.updated errors", async () => {
|
||||||
const sessionID = "quotio-both-message-updated"
|
const sessionID = "cliproxyapi-both-message-updated"
|
||||||
const harness = createHarness({ mode: "both" })
|
const harness = createHarness({ mode: "both" })
|
||||||
|
|
||||||
await primeMainSession(harness.eventHandler, sessionID)
|
await primeMainSession(harness.eventHandler, sessionID)
|
||||||
@@ -533,13 +501,7 @@ describe("Quotio-only fallback matrix", () => {
|
|||||||
expect(harness.abortCalls).toEqual([])
|
expect(harness.abortCalls).toEqual([])
|
||||||
expect(harness.promptCalls).toEqual([])
|
expect(harness.promptCalls).toEqual([])
|
||||||
expect(harness.promptAsyncCalls).toHaveLength(1)
|
expect(harness.promptAsyncCalls).toHaveLength(1)
|
||||||
expect(harness.promptAsyncCalls[0]?.model).toEqual({
|
expect(harness.promptAsyncCalls[0]?.model).toEqual(FIRST_FALLBACK_MODEL)
|
||||||
providerID: "quotio",
|
expect(output.message["model"]).toEqual(FIRST_FALLBACK_MODEL)
|
||||||
modelID: "claude-sonnet-4-6",
|
|
||||||
})
|
|
||||||
expect(output.message["model"]).toEqual({
|
|
||||||
providerID: "quotio",
|
|
||||||
modelID: "claude-sonnet-4-6",
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
Reference in New Issue
Block a user