From f0857a88fb5f23b3e59fdd77bc1919514c9248ef Mon Sep 17 00:00:00 2001 From: YeonGyu-Kim Date: Mon, 11 May 2026 12:45:51 +0900 Subject: [PATCH] test(ralph-loop): add dispatch-failure invariant tests Lock the contract that durable iteration state and visible UI must only\nadvance when the continuation dispatch is semantically accepted. Adds 4\npermanent invariant tests covering the idle, session.error retry, and\nverification-failure orchestration paths, plus the reset-strategy\nsilent-null path. --- .../dispatch-failure-invariant.test.ts | 254 ++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 src/hooks/ralph-loop/dispatch-failure-invariant.test.ts diff --git a/src/hooks/ralph-loop/dispatch-failure-invariant.test.ts b/src/hooks/ralph-loop/dispatch-failure-invariant.test.ts new file mode 100644 index 000000000..174ce459a --- /dev/null +++ b/src/hooks/ralph-loop/dispatch-failure-invariant.test.ts @@ -0,0 +1,254 @@ +/// +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { createRalphLoopHook } from "./index" +import { ULTRAWORK_VERIFICATION_PROMISE } from "./constants" +import { clearState, writeState } from "./storage" + +describe("ralph-loop dispatch failure invariants", () => { + const testDirectory = join(tmpdir(), `ralph-loop-dispatch-failure-${Date.now()}`) + let promptCalls: Array<{ sessionID: string; text: string }> + let toastCalls: Array<{ title: string; message: string; variant: string }> + let messagesCalls: Array<{ sessionID: string }> + let createSessionCalls: Array<{ parentID: string }> + + beforeEach(() => { + promptCalls = [] + toastCalls = [] + messagesCalls = [] + createSessionCalls = [] + mkdirSync(testDirectory, { recursive: true }) + clearState(testDirectory) + }) + + afterEach(() => { + clearState(testDirectory) + if (existsSync(testDirectory)) { + rmSync(testDirectory, { recursive: true, force: true }) + } + }) + + test("#given idle path #when promptAsync throws #then no state or toast advance", async () => { + // given + const hook = createRalphLoopHook({ + directory: testDirectory, + project: testDirectory, + worktree: testDirectory, + serverUrl: "http://localhost:4096", + $: async () => ({}), + client: { + session: { + messages: async (options: { path: { id: string } }) => { + messagesCalls.push({ sessionID: options.path.id }) + return { data: [] } + }, + promptAsync: async () => { + throw new Error("simulated dispatch failure") + }, + prompt: async () => ({}), + create: async () => ({ data: { id: "new-session-id" } }), + }, + tui: { + showToast: async (options: { body: { title: string; message: string; variant: string } }) => { + toastCalls.push(options.body) + return {} + }, + }, + }, + } as never) + + hook.startLoop("session-123", "Keep working", { + messageCountAtStart: 0, + maxIterations: 5, + }) + expect(hook.getState()?.iteration).toBe(1) + + // when + await hook.event({ + event: { type: "session.idle", properties: { sessionID: "session-123" } }, + }) + + // then + expect(toastCalls.some((toast) => toast.title === "Ralph Loop" && toast.message.includes("Iteration"))).toBe(false) + expect(hook.getState()).toBeNull() + expect(toastCalls.some((toast) => toast.title === "Ralph Loop Failed" && toast.message.includes("dispatch_rejected"))).toBe(true) + }) + + test("#given error retry path #when promptAsync throws #then no state or toast advance", async () => { + // given + const hook = createRalphLoopHook({ + directory: testDirectory, + project: testDirectory, + worktree: testDirectory, + serverUrl: "http://localhost:4096", + $: async () => ({}), + client: { + session: { + messages: async (options: { path: { id: string } }) => { + messagesCalls.push({ sessionID: options.path.id }) + return { data: [] } + }, + promptAsync: async () => { + throw new Error("simulated dispatch failure") + }, + prompt: async () => ({}), + create: async () => ({ data: { id: "new-session-id" } }), + }, + tui: { + showToast: async (options: { body: { title: string; message: string; variant: string } }) => { + toastCalls.push(options.body) + return {} + }, + }, + }, + } as never) + + hook.startLoop("session-123", "Keep working", { + messageCountAtStart: 0, + maxIterations: 5, + }) + expect(hook.getState()?.iteration).toBe(1) + + // when + await hook.event({ + event: { + type: "session.error", + properties: { + sessionID: "session-123", + error: { name: "RuntimeError" }, + }, + }, + }) + + // then + expect(toastCalls.some((toast) => toast.title === "Ralph Loop" && toast.message.includes("Iteration"))).toBe(false) + expect(hook.getState()).toBeNull() + expect(toastCalls.some((toast) => toast.title === "Ralph Loop Failed" && toast.message.includes("dispatch_rejected"))).toBe(true) + }) + + test("#given verification-failure path #when promptAsync throws #then iteration not advanced", async () => { + // given + const parentTranscriptPath = join(testDirectory, "transcript-parent.jsonl") + const oracleTranscriptPath = join(testDirectory, "transcript-oracle.jsonl") + const hook = createRalphLoopHook({ + directory: testDirectory, + project: testDirectory, + worktree: testDirectory, + serverUrl: "http://localhost:4096", + $: async () => ({}), + client: { + session: { + messages: async (options: { path: { id: string } }) => { + messagesCalls.push({ sessionID: options.path.id }) + if (options.path.id === "session-123") { + return { data: [{}, {}, {}] } + } + return { data: [] } + }, + promptAsync: async (options: { body: { parts: Array<{ type: string; text: string }> } }) => { + if (options.body.parts[0]?.text.includes("Verification failed")) { + throw new Error("simulated dispatch failure") + } + return {} + }, + prompt: async () => ({}), + abort: async () => ({}), + create: async () => ({ data: { id: "new-session-id" } }), + }, + tui: { + showToast: async (options: { body: { title: string; message: string; variant: string } }) => { + toastCalls.push(options.body) + return {} + }, + }, + }, + } as never, { + getTranscriptPath: (sessionID): string => sessionID === "ses-oracle" ? oracleTranscriptPath : parentTranscriptPath, + }) + + hook.startLoop("session-123", "Build API", { ultrawork: true }) + writeState(testDirectory, { + ...hook.getState()!, + iteration: 2, + verification_pending: true, + verification_session_id: "ses-oracle", + completion_promise: ULTRAWORK_VERIFICATION_PROMISE, + initial_completion_promise: "DONE", + }) + writeState(testDirectory, { + ...hook.getState()!, + verification_session_id: "ses-oracle", + }) + writeFileSync( + oracleTranscriptPath, + `${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "verification failed" } })}\n`, + ) + + const preRestartIteration = hook.getState()?.iteration + + // when + await hook.event({ event: { type: "session.idle", properties: { sessionID: "ses-oracle" } } }) + + // then + expect(preRestartIteration).toBe(2) + expect(hook.getState()).toBeNull() + expect(toastCalls.some((toast) => toast.title === "Ralph Loop Failed" && toast.message.includes("Verification continuation rejected"))).toBe(true) + }) + + test("#given reset strategy #when createIterationSession returns null #then dispatch failure surfaces", async () => { + // given + const hook = createRalphLoopHook({ + directory: testDirectory, + project: testDirectory, + worktree: testDirectory, + serverUrl: "http://localhost:4096", + $: async () => ({}), + client: { + session: { + messages: async (options: { path: { id: string } }) => { + messagesCalls.push({ sessionID: options.path.id }) + return { data: [] } + }, + promptAsync: async (options: { path: { id: string }; body: { parts: Array<{ type: string; text: string }> } }) => { + promptCalls.push({ + sessionID: options.path.id, + text: options.body.parts[0]?.text ?? "", + }) + return {} + }, + prompt: async () => ({}), + create: async (options: { body: { parentID: string } }) => { + createSessionCalls.push({ parentID: options.body.parentID }) + return { error: "fail", data: undefined } + }, + }, + tui: { + showToast: async (options: { body: { title: string; message: string; variant: string } }) => { + toastCalls.push(options.body) + return {} + }, + }, + }, + } as never) + + hook.startLoop("session-123", "Keep working", { + messageCountAtStart: 0, + maxIterations: 5, + strategy: "reset", + }) + expect(hook.getState()?.iteration).toBe(1) + + // when + await hook.event({ + event: { type: "session.idle", properties: { sessionID: "session-123" } }, + }) + + // then + expect(hook.getState()).toBeNull() + expect(promptCalls).toHaveLength(0) + expect(createSessionCalls).toHaveLength(1) + expect(toastCalls.some((toast) => toast.title === "Ralph Loop Failed" && toast.message.includes("session_creation_rejected"))).toBe(true) + }) +})