From ec552bcf34e749f5c7dc6d438af279452c2cfe5e Mon Sep 17 00:00:00 2001 From: YeonGyu-Kim Date: Tue, 26 May 2026 14:28:35 +0900 Subject: [PATCH] fix(ralph-loop): stop loop when latest assistant turn made no model progress Add no-progress turn detector that inspects the most recent assistant message: if finish is 'unknown', all token counts are zero, and there is no meaningful content beyond step-start/step-finish markers, the turn is classified as no-progress. Integrate the check at all three idle/completion/error continuation points in the event handler so the loop stops cleanly with a warning toast instead of injecting another internal prompt. --- .../ralph-loop/no-progress-loop-stop.test.ts | 93 ++++++++++++++ .../ralph-loop/no-progress-turn-detector.ts | 118 ++++++++++++++++++ .../ralph-loop/ralph-loop-event-handler.ts | 57 +++++++++ 3 files changed, 268 insertions(+) create mode 100644 src/hooks/ralph-loop/no-progress-loop-stop.test.ts create mode 100644 src/hooks/ralph-loop/no-progress-turn-detector.ts diff --git a/src/hooks/ralph-loop/no-progress-loop-stop.test.ts b/src/hooks/ralph-loop/no-progress-loop-stop.test.ts new file mode 100644 index 000000000..cdeb867d4 --- /dev/null +++ b/src/hooks/ralph-loop/no-progress-loop-stop.test.ts @@ -0,0 +1,93 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { existsSync, mkdirSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { unsafeTestValue } from "../../../test-support/unsafe-test-value" +import { createRalphLoopHook } from "./index" +import { clearState } from "./storage" + +type PromptCall = { + readonly sessionID: string + readonly text: string +} + +type ToastCall = { + readonly title: string + readonly message: string + readonly variant: string +} + +describe("ralph-loop no-progress stop", () => { + const testDirectory = join(tmpdir(), "ralph-loop-no-progress-" + Date.now()) + let promptCalls: PromptCall[] + let toastCalls: ToastCall[] + + beforeEach(() => { + promptCalls = [] + toastCalls = [] + if (!existsSync(testDirectory)) { + mkdirSync(testDirectory, { recursive: true }) + } + clearState(testDirectory) + }) + + afterEach(() => { + clearState(testDirectory) + if (existsSync(testDirectory)) { + rmSync(testDirectory, { recursive: true, force: true }) + } + }) + + test("#given latest assistant made no model progress #when session goes idle #then ralph loop stops without injecting another continuation", async () => { + const hook = createRalphLoopHook(unsafeTestValue({ + client: { + session: { + promptAsync: async (opts: { readonly path: { readonly id: string }; readonly body: { readonly parts: readonly [{ readonly text: string }] } }) => { + promptCalls.push({ + sessionID: opts.path.id, + text: opts.body.parts[0].text, + }) + return {} + }, + messages: async () => ({ + data: [ + { info: { role: "user" }, parts: [{ type: "text", text: "Continue" }] }, + { + info: { + role: "assistant", + finish: "unknown", + tokens: { + input: 0, + output: 0, + reasoning: 0, + cache: { write: 0, read: 0 }, + }, + }, + parts: [ + { type: "step-start" }, + { type: "step-finish" }, + ], + }, + ], + }), + }, + tui: { + showToast: async (opts: { readonly body: ToastCall }) => { + toastCalls.push(opts.body) + return {} + }, + }, + }, + directory: testDirectory, + })) + hook.startLoop("session-123", "Build API", { ultrawork: true }) + + await hook.event({ + event: { type: "session.idle", properties: { sessionID: "session-123" } }, + }) + + expect(promptCalls).toHaveLength(0) + expect(hook.getState()).toBeNull() + expect(toastCalls.some((toast) => toast.title === "Ralph Loop Stopped")).toBe(true) + }) +}) diff --git a/src/hooks/ralph-loop/no-progress-turn-detector.ts b/src/hooks/ralph-loop/no-progress-turn-detector.ts new file mode 100644 index 000000000..123211cb2 --- /dev/null +++ b/src/hooks/ralph-loop/no-progress-turn-detector.ts @@ -0,0 +1,118 @@ +import type { PluginInput } from "@opencode-ai/plugin" +import { isRecord, log, normalizeSDKResponse } from "../../shared" +import { withTimeout } from "./with-timeout" + +type SessionMessage = { + readonly info?: unknown + readonly role?: unknown + readonly parts?: readonly unknown[] + readonly finish?: unknown + readonly tokens?: unknown +} + +function getMessageRole(message: SessionMessage): string | undefined { + const info = isRecord(message.info) ? message.info : undefined + return typeof info?.role === "string" + ? info.role + : typeof message.role === "string" + ? message.role + : undefined +} + +function getMessageFinish(message: SessionMessage): string | undefined { + const info = isRecord(message.info) ? message.info : undefined + return typeof info?.finish === "string" + ? info.finish + : typeof message.finish === "string" + ? message.finish + : undefined +} + +function getTokenCount(tokens: unknown, key: "input" | "output" | "reasoning"): number | undefined { + if (!isRecord(tokens)) return undefined + const value = tokens[key] + return typeof value === "number" && Number.isFinite(value) ? value : undefined +} + +function getCacheTokenCount(tokens: unknown, key: "write" | "read"): number | undefined { + if (!isRecord(tokens)) return undefined + const cache = tokens.cache + if (!isRecord(cache)) return undefined + const value = cache[key] + return typeof value === "number" && Number.isFinite(value) ? value : undefined +} + +function getMessageTokens(message: SessionMessage): unknown { + const info = isRecord(message.info) ? message.info : undefined + return info?.tokens ?? message.tokens +} + +function allTokenCountsAreZero(message: SessionMessage): boolean { + const tokens = getMessageTokens(message) + const counts = [ + getTokenCount(tokens, "input"), + getTokenCount(tokens, "output"), + getTokenCount(tokens, "reasoning"), + getCacheTokenCount(tokens, "write"), + getCacheTokenCount(tokens, "read"), + ] + return counts.every((count) => count === 0) +} + +function partHasAssistantContent(part: unknown): boolean { + if (!isRecord(part)) return false + const type = typeof part.type === "string" ? part.type : undefined + if (type === "step-start" || type === "step-finish") return false + const text = typeof part.text === "string" ? part.text.trim() : "" + return type !== undefined || text.length > 0 +} + +function hasAssistantContent(message: SessionMessage): boolean { + return message.parts?.some(partHasAssistantContent) ?? false +} + +function isNoProgressAssistantMessage(message: SessionMessage): boolean { + return getMessageRole(message) === "assistant" + && getMessageFinish(message) === "unknown" + && allTokenCountsAreZero(message) + && !hasAssistantContent(message) +} + +export async function latestAssistantTurnMadeNoProgress( + ctx: PluginInput, + input: { + readonly sessionID: string + readonly directory: string + readonly apiTimeoutMs: number + readonly sinceMessageIndex?: number + }, +): Promise { + try { + const response = await withTimeout( + ctx.client.session.messages({ + path: { id: input.sessionID }, + query: { directory: input.directory }, + }), + input.apiTimeoutMs, + ) + const messages = normalizeSDKResponse(response, []) + const scopedMessages = + typeof input.sinceMessageIndex === "number" && input.sinceMessageIndex >= 0 + ? messages.slice(Math.min(input.sinceMessageIndex, messages.length)) + : messages + for (let index = scopedMessages.length - 1; index >= 0; index -= 1) { + const message = scopedMessages[index] + if (!message) continue + const role = getMessageRole(message) + if (role === "assistant") return isNoProgressAssistantMessage(message) + if (role === "user" || role === "tool") return false + } + return false + } catch (error) { + log("[ralph-loop] Failed to detect no-progress assistant turn", { + sessionID: input.sessionID, + error: String(error), + }) + return false + } +} diff --git a/src/hooks/ralph-loop/ralph-loop-event-handler.ts b/src/hooks/ralph-loop/ralph-loop-event-handler.ts index 9e5f73b92..302dc4b71 100644 --- a/src/hooks/ralph-loop/ralph-loop-event-handler.ts +++ b/src/hooks/ralph-loop/ralph-loop-event-handler.ts @@ -11,6 +11,7 @@ import { detectCompletionInTranscript, } from "./completion-promise-detector" import { continueIteration } from "./iteration-continuation" +import { latestAssistantTurnMadeNoProgress } from "./no-progress-turn-detector" import { handlePendingVerification } from "./pending-verification-handler" import { handleDeletedLoopSession, handleErroredLoopSession } from "./session-event-handler" @@ -258,6 +259,17 @@ function showIterationToast( }) } +function showNoProgressToast( + ctx: PluginInput, +): void { + showToastBestEffort(ctx, { + title: "Ralph Loop Stopped", + message: "Last assistant turn made no model progress; loop stopped to avoid repeated internal prompts.", + variant: "warning", + duration: 5000, + }) +} + export function createRalphLoopEventHandler( ctx: PluginInput, options: RalphLoopEventHandlerOptions, @@ -345,6 +357,21 @@ export function createRalphLoopEventHandler( return } + if (await latestAssistantTurnMadeNoProgress(ctx, { + sessionID, + directory: options.directory, + apiTimeoutMs: options.apiTimeoutMs, + sinceMessageIndex: state.message_count_at_start, + })) { + log(`[${HOOK_NAME}] Stopped after no-progress assistant turn`, { + sessionID, + iteration: state.iteration, + }) + options.loopState.clear() + showNoProgressToast(ctx) + return + } + if (state.verification_pending) { if (!verificationSessionID && matchesParentSession) { log(`[${HOOK_NAME}] Verification pending without tracked oracle session, running recovery check`, { @@ -423,6 +450,21 @@ export function createRalphLoopEventHandler( return } + if (await latestAssistantTurnMadeNoProgress(ctx, { + sessionID, + directory: options.directory, + apiTimeoutMs: options.apiTimeoutMs, + sinceMessageIndex: stateAfterSettle.message_count_at_start, + })) { + log(`[${HOOK_NAME}] Stopped after no-progress assistant turn`, { + sessionID, + iteration: stateAfterSettle.iteration, + }) + options.loopState.clear() + showNoProgressToast(ctx) + return + } + const nextIteration = stateAfterSettle.iteration + 1 const previewState: RalphLoopState = { ...stateAfterSettle, iteration: nextIteration } @@ -603,6 +645,21 @@ export function createRalphLoopEventHandler( return } + if (await latestAssistantTurnMadeNoProgress(ctx, { + sessionID, + directory: options.directory, + apiTimeoutMs: options.apiTimeoutMs, + sinceMessageIndex: stateAfterSettle.message_count_at_start, + })) { + log(`[${HOOK_NAME}] Stopped after no-progress assistant turn following runtime error`, { + sessionID, + iteration: stateAfterSettle.iteration, + }) + options.loopState.clear() + showNoProgressToast(ctx) + return + } + const nextIteration = stateAfterSettle.iteration + 1 const previewState: RalphLoopState = { ...stateAfterSettle, iteration: nextIteration }