fix(ralph-loop): stop loop when latest assistant turn made no model progress
Add no-progress turn detector that inspects the most recent assistant message: if finish is 'unknown', all token counts are zero, and there is no meaningful content beyond step-start/step-finish markers, the turn is classified as no-progress. Integrate the check at all three idle/completion/error continuation points in the event handler so the loop stops cleanly with a warning toast instead of injecting another internal prompt.
This commit is contained in:
YeonGyu-Kim
@@ -0,0 +1,93 @@
|
||||
import { afterEach, beforeEach, describe, expect, test } from "bun:test"
|
||||
import { existsSync, mkdirSync, rmSync } from "node:fs"
|
||||
import { tmpdir } from "node:os"
|
||||
import { join } from "node:path"
|
||||
import { unsafeTestValue } from "../../../test-support/unsafe-test-value"
|
||||
import { createRalphLoopHook } from "./index"
|
||||
import { clearState } from "./storage"
|
||||
|
||||
type PromptCall = {
|
||||
readonly sessionID: string
|
||||
readonly text: string
|
||||
}
|
||||
|
||||
type ToastCall = {
|
||||
readonly title: string
|
||||
readonly message: string
|
||||
readonly variant: string
|
||||
}
|
||||
|
||||
describe("ralph-loop no-progress stop", () => {
|
||||
const testDirectory = join(tmpdir(), "ralph-loop-no-progress-" + Date.now())
|
||||
let promptCalls: PromptCall[]
|
||||
let toastCalls: ToastCall[]
|
||||
|
||||
beforeEach(() => {
|
||||
promptCalls = []
|
||||
toastCalls = []
|
||||
if (!existsSync(testDirectory)) {
|
||||
mkdirSync(testDirectory, { recursive: true })
|
||||
}
|
||||
clearState(testDirectory)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
clearState(testDirectory)
|
||||
if (existsSync(testDirectory)) {
|
||||
rmSync(testDirectory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test("#given latest assistant made no model progress #when session goes idle #then ralph loop stops without injecting another continuation", async () => {
|
||||
const hook = createRalphLoopHook(unsafeTestValue({
|
||||
client: {
|
||||
session: {
|
||||
promptAsync: async (opts: { readonly path: { readonly id: string }; readonly body: { readonly parts: readonly [{ readonly text: string }] } }) => {
|
||||
promptCalls.push({
|
||||
sessionID: opts.path.id,
|
||||
text: opts.body.parts[0].text,
|
||||
})
|
||||
return {}
|
||||
},
|
||||
messages: async () => ({
|
||||
data: [
|
||||
{ info: { role: "user" }, parts: [{ type: "text", text: "Continue" }] },
|
||||
{
|
||||
info: {
|
||||
role: "assistant",
|
||||
finish: "unknown",
|
||||
tokens: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
reasoning: 0,
|
||||
cache: { write: 0, read: 0 },
|
||||
},
|
||||
},
|
||||
parts: [
|
||||
{ type: "step-start" },
|
||||
{ type: "step-finish" },
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
tui: {
|
||||
showToast: async (opts: { readonly body: ToastCall }) => {
|
||||
toastCalls.push(opts.body)
|
||||
return {}
|
||||
},
|
||||
},
|
||||
},
|
||||
directory: testDirectory,
|
||||
}))
|
||||
hook.startLoop("session-123", "Build API", { ultrawork: true })
|
||||
|
||||
await hook.event({
|
||||
event: { type: "session.idle", properties: { sessionID: "session-123" } },
|
||||
})
|
||||
|
||||
expect(promptCalls).toHaveLength(0)
|
||||
expect(hook.getState()).toBeNull()
|
||||
expect(toastCalls.some((toast) => toast.title === "Ralph Loop Stopped")).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,118 @@
|
||||
import type { PluginInput } from "@opencode-ai/plugin"
|
||||
import { isRecord, log, normalizeSDKResponse } from "../../shared"
|
||||
import { withTimeout } from "./with-timeout"
|
||||
|
||||
type SessionMessage = {
|
||||
readonly info?: unknown
|
||||
readonly role?: unknown
|
||||
readonly parts?: readonly unknown[]
|
||||
readonly finish?: unknown
|
||||
readonly tokens?: unknown
|
||||
}
|
||||
|
||||
function getMessageRole(message: SessionMessage): string | undefined {
|
||||
const info = isRecord(message.info) ? message.info : undefined
|
||||
return typeof info?.role === "string"
|
||||
? info.role
|
||||
: typeof message.role === "string"
|
||||
? message.role
|
||||
: undefined
|
||||
}
|
||||
|
||||
function getMessageFinish(message: SessionMessage): string | undefined {
|
||||
const info = isRecord(message.info) ? message.info : undefined
|
||||
return typeof info?.finish === "string"
|
||||
? info.finish
|
||||
: typeof message.finish === "string"
|
||||
? message.finish
|
||||
: undefined
|
||||
}
|
||||
|
||||
function getTokenCount(tokens: unknown, key: "input" | "output" | "reasoning"): number | undefined {
|
||||
if (!isRecord(tokens)) return undefined
|
||||
const value = tokens[key]
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : undefined
|
||||
}
|
||||
|
||||
function getCacheTokenCount(tokens: unknown, key: "write" | "read"): number | undefined {
|
||||
if (!isRecord(tokens)) return undefined
|
||||
const cache = tokens.cache
|
||||
if (!isRecord(cache)) return undefined
|
||||
const value = cache[key]
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : undefined
|
||||
}
|
||||
|
||||
function getMessageTokens(message: SessionMessage): unknown {
|
||||
const info = isRecord(message.info) ? message.info : undefined
|
||||
return info?.tokens ?? message.tokens
|
||||
}
|
||||
|
||||
function allTokenCountsAreZero(message: SessionMessage): boolean {
|
||||
const tokens = getMessageTokens(message)
|
||||
const counts = [
|
||||
getTokenCount(tokens, "input"),
|
||||
getTokenCount(tokens, "output"),
|
||||
getTokenCount(tokens, "reasoning"),
|
||||
getCacheTokenCount(tokens, "write"),
|
||||
getCacheTokenCount(tokens, "read"),
|
||||
]
|
||||
return counts.every((count) => count === 0)
|
||||
}
|
||||
|
||||
function partHasAssistantContent(part: unknown): boolean {
|
||||
if (!isRecord(part)) return false
|
||||
const type = typeof part.type === "string" ? part.type : undefined
|
||||
if (type === "step-start" || type === "step-finish") return false
|
||||
const text = typeof part.text === "string" ? part.text.trim() : ""
|
||||
return type !== undefined || text.length > 0
|
||||
}
|
||||
|
||||
function hasAssistantContent(message: SessionMessage): boolean {
|
||||
return message.parts?.some(partHasAssistantContent) ?? false
|
||||
}
|
||||
|
||||
function isNoProgressAssistantMessage(message: SessionMessage): boolean {
|
||||
return getMessageRole(message) === "assistant"
|
||||
&& getMessageFinish(message) === "unknown"
|
||||
&& allTokenCountsAreZero(message)
|
||||
&& !hasAssistantContent(message)
|
||||
}
|
||||
|
||||
export async function latestAssistantTurnMadeNoProgress(
|
||||
ctx: PluginInput,
|
||||
input: {
|
||||
readonly sessionID: string
|
||||
readonly directory: string
|
||||
readonly apiTimeoutMs: number
|
||||
readonly sinceMessageIndex?: number
|
||||
},
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const response = await withTimeout(
|
||||
ctx.client.session.messages({
|
||||
path: { id: input.sessionID },
|
||||
query: { directory: input.directory },
|
||||
}),
|
||||
input.apiTimeoutMs,
|
||||
)
|
||||
const messages = normalizeSDKResponse<readonly SessionMessage[]>(response, [])
|
||||
const scopedMessages =
|
||||
typeof input.sinceMessageIndex === "number" && input.sinceMessageIndex >= 0
|
||||
? messages.slice(Math.min(input.sinceMessageIndex, messages.length))
|
||||
: messages
|
||||
for (let index = scopedMessages.length - 1; index >= 0; index -= 1) {
|
||||
const message = scopedMessages[index]
|
||||
if (!message) continue
|
||||
const role = getMessageRole(message)
|
||||
if (role === "assistant") return isNoProgressAssistantMessage(message)
|
||||
if (role === "user" || role === "tool") return false
|
||||
}
|
||||
return false
|
||||
} catch (error) {
|
||||
log("[ralph-loop] Failed to detect no-progress assistant turn", {
|
||||
sessionID: input.sessionID,
|
||||
error: String(error),
|
||||
})
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
detectCompletionInTranscript,
|
||||
} from "./completion-promise-detector"
|
||||
import { continueIteration } from "./iteration-continuation"
|
||||
import { latestAssistantTurnMadeNoProgress } from "./no-progress-turn-detector"
|
||||
import { handlePendingVerification } from "./pending-verification-handler"
|
||||
import { handleDeletedLoopSession, handleErroredLoopSession } from "./session-event-handler"
|
||||
|
||||
@@ -258,6 +259,17 @@ function showIterationToast(
|
||||
})
|
||||
}
|
||||
|
||||
function showNoProgressToast(
|
||||
ctx: PluginInput,
|
||||
): void {
|
||||
showToastBestEffort(ctx, {
|
||||
title: "Ralph Loop Stopped",
|
||||
message: "Last assistant turn made no model progress; loop stopped to avoid repeated internal prompts.",
|
||||
variant: "warning",
|
||||
duration: 5000,
|
||||
})
|
||||
}
|
||||
|
||||
export function createRalphLoopEventHandler(
|
||||
ctx: PluginInput,
|
||||
options: RalphLoopEventHandlerOptions,
|
||||
@@ -345,6 +357,21 @@ export function createRalphLoopEventHandler(
|
||||
return
|
||||
}
|
||||
|
||||
if (await latestAssistantTurnMadeNoProgress(ctx, {
|
||||
sessionID,
|
||||
directory: options.directory,
|
||||
apiTimeoutMs: options.apiTimeoutMs,
|
||||
sinceMessageIndex: state.message_count_at_start,
|
||||
})) {
|
||||
log(`[${HOOK_NAME}] Stopped after no-progress assistant turn`, {
|
||||
sessionID,
|
||||
iteration: state.iteration,
|
||||
})
|
||||
options.loopState.clear()
|
||||
showNoProgressToast(ctx)
|
||||
return
|
||||
}
|
||||
|
||||
if (state.verification_pending) {
|
||||
if (!verificationSessionID && matchesParentSession) {
|
||||
log(`[${HOOK_NAME}] Verification pending without tracked oracle session, running recovery check`, {
|
||||
@@ -423,6 +450,21 @@ export function createRalphLoopEventHandler(
|
||||
return
|
||||
}
|
||||
|
||||
if (await latestAssistantTurnMadeNoProgress(ctx, {
|
||||
sessionID,
|
||||
directory: options.directory,
|
||||
apiTimeoutMs: options.apiTimeoutMs,
|
||||
sinceMessageIndex: stateAfterSettle.message_count_at_start,
|
||||
})) {
|
||||
log(`[${HOOK_NAME}] Stopped after no-progress assistant turn`, {
|
||||
sessionID,
|
||||
iteration: stateAfterSettle.iteration,
|
||||
})
|
||||
options.loopState.clear()
|
||||
showNoProgressToast(ctx)
|
||||
return
|
||||
}
|
||||
|
||||
const nextIteration = stateAfterSettle.iteration + 1
|
||||
const previewState: RalphLoopState = { ...stateAfterSettle, iteration: nextIteration }
|
||||
|
||||
@@ -603,6 +645,21 @@ export function createRalphLoopEventHandler(
|
||||
return
|
||||
}
|
||||
|
||||
if (await latestAssistantTurnMadeNoProgress(ctx, {
|
||||
sessionID,
|
||||
directory: options.directory,
|
||||
apiTimeoutMs: options.apiTimeoutMs,
|
||||
sinceMessageIndex: stateAfterSettle.message_count_at_start,
|
||||
})) {
|
||||
log(`[${HOOK_NAME}] Stopped after no-progress assistant turn following runtime error`, {
|
||||
sessionID,
|
||||
iteration: stateAfterSettle.iteration,
|
||||
})
|
||||
options.loopState.clear()
|
||||
showNoProgressToast(ctx)
|
||||
return
|
||||
}
|
||||
|
||||
const nextIteration = stateAfterSettle.iteration + 1
|
||||
const previewState: RalphLoopState = { ...stateAfterSettle, iteration: nextIteration }
|
||||
|
||||
|
||||
Reference in New Issue
Block a user