fix(ralph-loop): stop loop when latest assistant turn made no model progress
Add no-progress turn detector that inspects the most recent assistant message: if finish is 'unknown', all token counts are zero, and there is no meaningful content beyond step-start/step-finish markers, the turn is classified as no-progress. Integrate the check at all three idle/completion/error continuation points in the event handler so the loop stops cleanly with a warning toast instead of injecting another internal prompt.
This commit is contained in:
@@ -0,0 +1,93 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, test } from "bun:test"
|
||||||
|
import { existsSync, mkdirSync, rmSync } from "node:fs"
|
||||||
|
import { tmpdir } from "node:os"
|
||||||
|
import { join } from "node:path"
|
||||||
|
import { unsafeTestValue } from "../../../test-support/unsafe-test-value"
|
||||||
|
import { createRalphLoopHook } from "./index"
|
||||||
|
import { clearState } from "./storage"
|
||||||
|
|
||||||
|
type PromptCall = {
|
||||||
|
readonly sessionID: string
|
||||||
|
readonly text: string
|
||||||
|
}
|
||||||
|
|
||||||
|
type ToastCall = {
|
||||||
|
readonly title: string
|
||||||
|
readonly message: string
|
||||||
|
readonly variant: string
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("ralph-loop no-progress stop", () => {
|
||||||
|
const testDirectory = join(tmpdir(), "ralph-loop-no-progress-" + Date.now())
|
||||||
|
let promptCalls: PromptCall[]
|
||||||
|
let toastCalls: ToastCall[]
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
promptCalls = []
|
||||||
|
toastCalls = []
|
||||||
|
if (!existsSync(testDirectory)) {
|
||||||
|
mkdirSync(testDirectory, { recursive: true })
|
||||||
|
}
|
||||||
|
clearState(testDirectory)
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
clearState(testDirectory)
|
||||||
|
if (existsSync(testDirectory)) {
|
||||||
|
rmSync(testDirectory, { recursive: true, force: true })
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test("#given latest assistant made no model progress #when session goes idle #then ralph loop stops without injecting another continuation", async () => {
|
||||||
|
const hook = createRalphLoopHook(unsafeTestValue({
|
||||||
|
client: {
|
||||||
|
session: {
|
||||||
|
promptAsync: async (opts: { readonly path: { readonly id: string }; readonly body: { readonly parts: readonly [{ readonly text: string }] } }) => {
|
||||||
|
promptCalls.push({
|
||||||
|
sessionID: opts.path.id,
|
||||||
|
text: opts.body.parts[0].text,
|
||||||
|
})
|
||||||
|
return {}
|
||||||
|
},
|
||||||
|
messages: async () => ({
|
||||||
|
data: [
|
||||||
|
{ info: { role: "user" }, parts: [{ type: "text", text: "Continue" }] },
|
||||||
|
{
|
||||||
|
info: {
|
||||||
|
role: "assistant",
|
||||||
|
finish: "unknown",
|
||||||
|
tokens: {
|
||||||
|
input: 0,
|
||||||
|
output: 0,
|
||||||
|
reasoning: 0,
|
||||||
|
cache: { write: 0, read: 0 },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
parts: [
|
||||||
|
{ type: "step-start" },
|
||||||
|
{ type: "step-finish" },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
tui: {
|
||||||
|
showToast: async (opts: { readonly body: ToastCall }) => {
|
||||||
|
toastCalls.push(opts.body)
|
||||||
|
return {}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
directory: testDirectory,
|
||||||
|
}))
|
||||||
|
hook.startLoop("session-123", "Build API", { ultrawork: true })
|
||||||
|
|
||||||
|
await hook.event({
|
||||||
|
event: { type: "session.idle", properties: { sessionID: "session-123" } },
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(promptCalls).toHaveLength(0)
|
||||||
|
expect(hook.getState()).toBeNull()
|
||||||
|
expect(toastCalls.some((toast) => toast.title === "Ralph Loop Stopped")).toBe(true)
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
import type { PluginInput } from "@opencode-ai/plugin"
|
||||||
|
import { isRecord, log, normalizeSDKResponse } from "../../shared"
|
||||||
|
import { withTimeout } from "./with-timeout"
|
||||||
|
|
||||||
|
type SessionMessage = {
|
||||||
|
readonly info?: unknown
|
||||||
|
readonly role?: unknown
|
||||||
|
readonly parts?: readonly unknown[]
|
||||||
|
readonly finish?: unknown
|
||||||
|
readonly tokens?: unknown
|
||||||
|
}
|
||||||
|
|
||||||
|
function getMessageRole(message: SessionMessage): string | undefined {
|
||||||
|
const info = isRecord(message.info) ? message.info : undefined
|
||||||
|
return typeof info?.role === "string"
|
||||||
|
? info.role
|
||||||
|
: typeof message.role === "string"
|
||||||
|
? message.role
|
||||||
|
: undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
function getMessageFinish(message: SessionMessage): string | undefined {
|
||||||
|
const info = isRecord(message.info) ? message.info : undefined
|
||||||
|
return typeof info?.finish === "string"
|
||||||
|
? info.finish
|
||||||
|
: typeof message.finish === "string"
|
||||||
|
? message.finish
|
||||||
|
: undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
function getTokenCount(tokens: unknown, key: "input" | "output" | "reasoning"): number | undefined {
|
||||||
|
if (!isRecord(tokens)) return undefined
|
||||||
|
const value = tokens[key]
|
||||||
|
return typeof value === "number" && Number.isFinite(value) ? value : undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
function getCacheTokenCount(tokens: unknown, key: "write" | "read"): number | undefined {
|
||||||
|
if (!isRecord(tokens)) return undefined
|
||||||
|
const cache = tokens.cache
|
||||||
|
if (!isRecord(cache)) return undefined
|
||||||
|
const value = cache[key]
|
||||||
|
return typeof value === "number" && Number.isFinite(value) ? value : undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
function getMessageTokens(message: SessionMessage): unknown {
|
||||||
|
const info = isRecord(message.info) ? message.info : undefined
|
||||||
|
return info?.tokens ?? message.tokens
|
||||||
|
}
|
||||||
|
|
||||||
|
function allTokenCountsAreZero(message: SessionMessage): boolean {
|
||||||
|
const tokens = getMessageTokens(message)
|
||||||
|
const counts = [
|
||||||
|
getTokenCount(tokens, "input"),
|
||||||
|
getTokenCount(tokens, "output"),
|
||||||
|
getTokenCount(tokens, "reasoning"),
|
||||||
|
getCacheTokenCount(tokens, "write"),
|
||||||
|
getCacheTokenCount(tokens, "read"),
|
||||||
|
]
|
||||||
|
return counts.every((count) => count === 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
function partHasAssistantContent(part: unknown): boolean {
|
||||||
|
if (!isRecord(part)) return false
|
||||||
|
const type = typeof part.type === "string" ? part.type : undefined
|
||||||
|
if (type === "step-start" || type === "step-finish") return false
|
||||||
|
const text = typeof part.text === "string" ? part.text.trim() : ""
|
||||||
|
return type !== undefined || text.length > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasAssistantContent(message: SessionMessage): boolean {
|
||||||
|
return message.parts?.some(partHasAssistantContent) ?? false
|
||||||
|
}
|
||||||
|
|
||||||
|
function isNoProgressAssistantMessage(message: SessionMessage): boolean {
|
||||||
|
return getMessageRole(message) === "assistant"
|
||||||
|
&& getMessageFinish(message) === "unknown"
|
||||||
|
&& allTokenCountsAreZero(message)
|
||||||
|
&& !hasAssistantContent(message)
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function latestAssistantTurnMadeNoProgress(
|
||||||
|
ctx: PluginInput,
|
||||||
|
input: {
|
||||||
|
readonly sessionID: string
|
||||||
|
readonly directory: string
|
||||||
|
readonly apiTimeoutMs: number
|
||||||
|
readonly sinceMessageIndex?: number
|
||||||
|
},
|
||||||
|
): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
const response = await withTimeout(
|
||||||
|
ctx.client.session.messages({
|
||||||
|
path: { id: input.sessionID },
|
||||||
|
query: { directory: input.directory },
|
||||||
|
}),
|
||||||
|
input.apiTimeoutMs,
|
||||||
|
)
|
||||||
|
const messages = normalizeSDKResponse<readonly SessionMessage[]>(response, [])
|
||||||
|
const scopedMessages =
|
||||||
|
typeof input.sinceMessageIndex === "number" && input.sinceMessageIndex >= 0
|
||||||
|
? messages.slice(Math.min(input.sinceMessageIndex, messages.length))
|
||||||
|
: messages
|
||||||
|
for (let index = scopedMessages.length - 1; index >= 0; index -= 1) {
|
||||||
|
const message = scopedMessages[index]
|
||||||
|
if (!message) continue
|
||||||
|
const role = getMessageRole(message)
|
||||||
|
if (role === "assistant") return isNoProgressAssistantMessage(message)
|
||||||
|
if (role === "user" || role === "tool") return false
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
} catch (error) {
|
||||||
|
log("[ralph-loop] Failed to detect no-progress assistant turn", {
|
||||||
|
sessionID: input.sessionID,
|
||||||
|
error: String(error),
|
||||||
|
})
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,6 +11,7 @@ import {
|
|||||||
detectCompletionInTranscript,
|
detectCompletionInTranscript,
|
||||||
} from "./completion-promise-detector"
|
} from "./completion-promise-detector"
|
||||||
import { continueIteration } from "./iteration-continuation"
|
import { continueIteration } from "./iteration-continuation"
|
||||||
|
import { latestAssistantTurnMadeNoProgress } from "./no-progress-turn-detector"
|
||||||
import { handlePendingVerification } from "./pending-verification-handler"
|
import { handlePendingVerification } from "./pending-verification-handler"
|
||||||
import { handleDeletedLoopSession, handleErroredLoopSession } from "./session-event-handler"
|
import { handleDeletedLoopSession, handleErroredLoopSession } from "./session-event-handler"
|
||||||
|
|
||||||
@@ -258,6 +259,17 @@ function showIterationToast(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function showNoProgressToast(
|
||||||
|
ctx: PluginInput,
|
||||||
|
): void {
|
||||||
|
showToastBestEffort(ctx, {
|
||||||
|
title: "Ralph Loop Stopped",
|
||||||
|
message: "Last assistant turn made no model progress; loop stopped to avoid repeated internal prompts.",
|
||||||
|
variant: "warning",
|
||||||
|
duration: 5000,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
export function createRalphLoopEventHandler(
|
export function createRalphLoopEventHandler(
|
||||||
ctx: PluginInput,
|
ctx: PluginInput,
|
||||||
options: RalphLoopEventHandlerOptions,
|
options: RalphLoopEventHandlerOptions,
|
||||||
@@ -345,6 +357,21 @@ export function createRalphLoopEventHandler(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (await latestAssistantTurnMadeNoProgress(ctx, {
|
||||||
|
sessionID,
|
||||||
|
directory: options.directory,
|
||||||
|
apiTimeoutMs: options.apiTimeoutMs,
|
||||||
|
sinceMessageIndex: state.message_count_at_start,
|
||||||
|
})) {
|
||||||
|
log(`[${HOOK_NAME}] Stopped after no-progress assistant turn`, {
|
||||||
|
sessionID,
|
||||||
|
iteration: state.iteration,
|
||||||
|
})
|
||||||
|
options.loopState.clear()
|
||||||
|
showNoProgressToast(ctx)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if (state.verification_pending) {
|
if (state.verification_pending) {
|
||||||
if (!verificationSessionID && matchesParentSession) {
|
if (!verificationSessionID && matchesParentSession) {
|
||||||
log(`[${HOOK_NAME}] Verification pending without tracked oracle session, running recovery check`, {
|
log(`[${HOOK_NAME}] Verification pending without tracked oracle session, running recovery check`, {
|
||||||
@@ -423,6 +450,21 @@ export function createRalphLoopEventHandler(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (await latestAssistantTurnMadeNoProgress(ctx, {
|
||||||
|
sessionID,
|
||||||
|
directory: options.directory,
|
||||||
|
apiTimeoutMs: options.apiTimeoutMs,
|
||||||
|
sinceMessageIndex: stateAfterSettle.message_count_at_start,
|
||||||
|
})) {
|
||||||
|
log(`[${HOOK_NAME}] Stopped after no-progress assistant turn`, {
|
||||||
|
sessionID,
|
||||||
|
iteration: stateAfterSettle.iteration,
|
||||||
|
})
|
||||||
|
options.loopState.clear()
|
||||||
|
showNoProgressToast(ctx)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
const nextIteration = stateAfterSettle.iteration + 1
|
const nextIteration = stateAfterSettle.iteration + 1
|
||||||
const previewState: RalphLoopState = { ...stateAfterSettle, iteration: nextIteration }
|
const previewState: RalphLoopState = { ...stateAfterSettle, iteration: nextIteration }
|
||||||
|
|
||||||
@@ -603,6 +645,21 @@ export function createRalphLoopEventHandler(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (await latestAssistantTurnMadeNoProgress(ctx, {
|
||||||
|
sessionID,
|
||||||
|
directory: options.directory,
|
||||||
|
apiTimeoutMs: options.apiTimeoutMs,
|
||||||
|
sinceMessageIndex: stateAfterSettle.message_count_at_start,
|
||||||
|
})) {
|
||||||
|
log(`[${HOOK_NAME}] Stopped after no-progress assistant turn following runtime error`, {
|
||||||
|
sessionID,
|
||||||
|
iteration: stateAfterSettle.iteration,
|
||||||
|
})
|
||||||
|
options.loopState.clear()
|
||||||
|
showNoProgressToast(ctx)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
const nextIteration = stateAfterSettle.iteration + 1
|
const nextIteration = stateAfterSettle.iteration + 1
|
||||||
const previewState: RalphLoopState = { ...stateAfterSettle, iteration: nextIteration }
|
const previewState: RalphLoopState = { ...stateAfterSettle, iteration: nextIteration }
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user