diff --git a/src/hooks/ralph-loop/completion-promise-detector-test-input.ts b/src/hooks/ralph-loop/completion-promise-detector-test-input.ts new file mode 100644 index 000000000..4a1e6c259 --- /dev/null +++ b/src/hooks/ralph-loop/completion-promise-detector-test-input.ts @@ -0,0 +1,23 @@ +/// +import type { PluginInput } from "@opencode-ai/plugin" + +export type SessionMessage = { + info?: { role?: string } + parts?: Array<{ type: string; text?: string }> +} + +export function createPluginInput(messages: SessionMessage[]): PluginInput { + const pluginInput = { + client: { session: {} } as PluginInput["client"], + project: {} as PluginInput["project"], + directory: "/tmp", + worktree: "/tmp", + serverUrl: new URL("http://localhost"), + $: {} as PluginInput["$"], + } as PluginInput + + pluginInput.client.session.messages = + (async () => ({ data: messages })) as unknown as PluginInput["client"]["session"]["messages"] + + return pluginInput +} diff --git a/src/hooks/ralph-loop/completion-promise-detector.test.ts b/src/hooks/ralph-loop/completion-promise-detector.test.ts index 0e87bf94e..317883a6c 100644 --- a/src/hooks/ralph-loop/completion-promise-detector.test.ts +++ b/src/hooks/ralph-loop/completion-promise-detector.test.ts @@ -1,34 +1,13 @@ /// import { describe, expect, test } from "bun:test" -import type { PluginInput } from "@opencode-ai/plugin" -import { detectCompletionInSessionMessages, detectSemanticCompletion } from "./completion-promise-detector" - -type SessionMessage = { - info?: { role?: string } - parts?: Array<{ type: string; text?: string }> -} - -function createPluginInput(messages: SessionMessage[]): PluginInput { - const pluginInput = { - client: { session: {} } as PluginInput["client"], - project: {} as PluginInput["project"], - directory: "/tmp", - worktree: "/tmp", - serverUrl: new URL("http://localhost"), - $: {} as PluginInput["$"], - } as PluginInput - - pluginInput.client.session.messages = - (async () => ({ data: messages })) as unknown as PluginInput["client"]["session"]["messages"] - - return pluginInput -} +import { detectCompletionInSessionMessages } from "./completion-promise-detector" +import { createPluginInput } from "./completion-promise-detector-test-input" describe("detectCompletionInSessionMessages", () => { describe("#given session with prior DONE and new messages", () => { test("#when sinceMessageIndex excludes prior DONE #then should NOT detect completion", async () => { // #given - const messages: SessionMessage[] = [ + const messages = [ { info: { role: "assistant" }, parts: [{ type: "text", text: "Old completion DONE" }], @@ -55,7 +34,7 @@ describe("detectCompletionInSessionMessages", () => { test("#when sinceMessageIndex includes current DONE #then should detect completion", async () => { // #given - const messages: SessionMessage[] = [ + const messages = [ { info: { role: "assistant" }, parts: [{ type: "text", text: "Old completion DONE" }], @@ -84,7 +63,7 @@ describe("detectCompletionInSessionMessages", () => { describe("#given no sinceMessageIndex (backward compat)", () => { test("#then should scan all messages", async () => { // #given - const messages: SessionMessage[] = [ + const messages = [ { info: { role: "assistant" }, parts: [{ type: "text", text: "Old completion DONE" }], @@ -111,7 +90,7 @@ describe("detectCompletionInSessionMessages", () => { describe("#given promise appears in tool_result part (not text part)", () => { test("#when Oracle returns VERIFIED via task() tool_result #then should NOT detect completion", async () => { - const messages: SessionMessage[] = [ + const messages = [ { info: { role: "assistant" }, parts: [ @@ -141,7 +120,7 @@ describe("detectCompletionInSessionMessages", () => { }) test("#when DONE appears only in tool_result part #then should NOT detect completion", async () => { - const messages: SessionMessage[] = [ + const messages = [ { info: { role: "assistant" }, parts: [ @@ -163,7 +142,7 @@ describe("detectCompletionInSessionMessages", () => { }) test("#when promise appears in tool_use part (not tool_result) #then should NOT detect completion", async () => { - const messages: SessionMessage[] = [ + const messages = [ { info: { role: "assistant" }, parts: [ @@ -184,217 +163,4 @@ describe("detectCompletionInSessionMessages", () => { expect(detected).toBe(false) }) }) - - describe("#given semantic completion patterns", () => { - test("#when agent says 'task is complete' without explicit promise #then should NOT detect completion", async () => { - // #given - const messages: SessionMessage[] = [ - { - info: { role: "assistant" }, - parts: [{ type: "text", text: "The task is complete. All work has been finished." }], - }, - ] - const ctx = createPluginInput(messages) - - // #when - const detected = await detectCompletionInSessionMessages(ctx, { - sessionID: "session-123", - promise: "DONE", - apiTimeoutMs: 1000, - directory: "/tmp", - }) - - // #then - expect(detected).toBe(false) - }) - - test("#when agent says 'all items are done' without explicit promise #then should NOT detect completion", async () => { - // #given - const messages: SessionMessage[] = [ - { - info: { role: "assistant" }, - parts: [{ type: "text", text: "All items are done and marked as complete." }], - }, - ] - const ctx = createPluginInput(messages) - - // #when - const detected = await detectCompletionInSessionMessages(ctx, { - sessionID: "session-123", - promise: "DONE", - apiTimeoutMs: 1000, - directory: "/tmp", - }) - - // #then - expect(detected).toBe(false) - }) - - test("#when agent says 'nothing left to do' without explicit promise #then should NOT detect completion", async () => { - // #given - const messages: SessionMessage[] = [ - { - info: { role: "assistant" }, - parts: [{ type: "text", text: "There is nothing left to do. Everything is finished." }], - }, - ] - const ctx = createPluginInput(messages) - - // #when - const detected = await detectCompletionInSessionMessages(ctx, { - sessionID: "session-123", - promise: "DONE", - apiTimeoutMs: 1000, - directory: "/tmp", - }) - - // #then - expect(detected).toBe(false) - }) - - test("#when agent says 'successfully completed all' without explicit promise #then should NOT detect completion", async () => { - // #given - const messages: SessionMessage[] = [ - { - info: { role: "assistant" }, - parts: [{ type: "text", text: "I have successfully completed all the required tasks." }], - }, - ] - const ctx = createPluginInput(messages) - - // #when - const detected = await detectCompletionInSessionMessages(ctx, { - sessionID: "session-123", - promise: "DONE", - apiTimeoutMs: 1000, - directory: "/tmp", - }) - - // #then - expect(detected).toBe(false) - }) - - test("#when promise is VERIFIED #then semantic completion should NOT trigger", async () => { - // #given - const messages: SessionMessage[] = [ - { - info: { role: "assistant" }, - parts: [{ type: "text", text: "The task is complete. All work has been finished." }], - }, - ] - const ctx = createPluginInput(messages) - - // #when - const detected = await detectCompletionInSessionMessages(ctx, { - sessionID: "session-123", - promise: "VERIFIED", - apiTimeoutMs: 1000, - directory: "/tmp", - }) - - // #then - expect(detected).toBe(false) - }) - - test("#when completion text appears inside a quote #then should NOT detect completion", async () => { - // #given - const messages: SessionMessage[] = [ - { - info: { role: "assistant" }, - parts: [{ type: "text", text: 'The user wrote: "the task is complete". I am still working.' }], - }, - ] - const ctx = createPluginInput(messages) - - // #when - const detected = await detectCompletionInSessionMessages(ctx, { - sessionID: "session-quoted", - promise: "DONE", - apiTimeoutMs: 1000, - directory: "/tmp", - }) - - // #then - expect(detected).toBe(false) - }) - - test("#when tool_result says all items are complete #then should NOT detect completion", async () => { - // #given - const messages: SessionMessage[] = [ - { - info: { role: "assistant" }, - parts: [ - { type: "tool_result", text: "Background agent report: all items are complete." }, - { type: "text", text: "Still validating the final behavior." }, - ], - }, - ] - const ctx = createPluginInput(messages) - - // #when - const detected = await detectCompletionInSessionMessages(ctx, { - sessionID: "session-tool-result-semantic", - promise: "DONE", - apiTimeoutMs: 1000, - directory: "/tmp", - }) - - // #then - expect(detected).toBe(false) - }) - - test("#when assistant says complete but not actually done #then should NOT detect completion", async () => { - // #given - const messages: SessionMessage[] = [ - { - info: { role: "assistant" }, - parts: [{ type: "text", text: "The implementation looks complete, but I still need to run the tests." }], - }, - ] - const ctx = createPluginInput(messages) - - // #when - const detected = await detectCompletionInSessionMessages(ctx, { - sessionID: "session-not-actually-done", - promise: "DONE", - apiTimeoutMs: 1000, - directory: "/tmp", - }) - - // #then - expect(detected).toBe(false) - }) - }) -}) - -describe("detectSemanticCompletion", () => { - describe("#given semantic completion patterns", () => { - test("#when text contains 'task is complete' #then should return true", () => { - expect(detectSemanticCompletion("The task is complete.")).toBe(true) - }) - - test("#when text contains 'all items are done' #then should return true", () => { - expect(detectSemanticCompletion("All items are done.")).toBe(true) - }) - - test("#when text contains 'nothing left to do' #then should return true", () => { - expect(detectSemanticCompletion("There is nothing left to do.")).toBe(true) - }) - - test("#when text contains 'successfully completed all' #then should return true", () => { - expect(detectSemanticCompletion("Successfully completed all tasks.")).toBe(true) - }) - - test("#when text contains 'everything is finished' #then should return true", () => { - expect(detectSemanticCompletion("Everything is finished.")).toBe(true) - }) - - test("#when text does NOT contain completion patterns #then should return false", () => { - expect(detectSemanticCompletion("Working on the next task.")).toBe(false) - }) - - test("#when text is empty #then should return false", () => { - expect(detectSemanticCompletion("")).toBe(false) - }) - }) }) diff --git a/src/hooks/ralph-loop/completion-promise-detector.ts b/src/hooks/ralph-loop/completion-promise-detector.ts index e02cd58ec..35b781a07 100644 --- a/src/hooks/ralph-loop/completion-promise-detector.ts +++ b/src/hooks/ralph-loop/completion-promise-detector.ts @@ -31,20 +31,6 @@ function buildPromisePattern(promise: string): RegExp { return new RegExp(`\\s*${escapeRegex(promise)}\\s*`, "is") } -const SEMANTIC_COMPLETION_PATTERNS = [ - /\b(?:task|work|implementation|all\s+tasks?)\s+(?:is|are)\s+(?:complete|completed|done|finished)\b/i, - /\ball\s+(?:items?|todos?|steps?)\s+(?:are\s+)?(?:complete|completed|done|finished|marked)\b/i, - /\b(?:everything|all\s+work)\s+(?:is\s+)?(?:complete|completed|done|finished)\b/i, - /\bsuccessfully\s+completed?\s+all\b/i, - /\bnothing\s+(?:left|more|remaining)\s+to\s+(?:do|implement|fix)\b/i, -] - -const SEMANTIC_DONE_FALLBACK_ENABLED = false - -export function detectSemanticCompletion(text: string): boolean { - return SEMANTIC_COMPLETION_PATTERNS.some((pattern) => pattern.test(text)) -} - export function detectCompletionInTranscript( transcriptPath: string | undefined, promise: string, @@ -57,7 +43,7 @@ export function detectCompletionInTranscript( const content = readFileSync(transcriptPath, "utf-8") const pattern = buildPromisePattern(promise) - const lines = content.split("\n").filter((line) => line.trim()) + const lines = content.split("\n").filter((line: string) => line.trim()) for (const line of lines) { try { @@ -68,11 +54,6 @@ export function detectCompletionInTranscript( const entryText = extractTranscriptEntryText(entry) if (!entryText) continue if (pattern.test(entryText)) return true - const isAssistantEntry = entry.type === "assistant" || entry.type === "text" - if (SEMANTIC_DONE_FALLBACK_ENABLED && promise === "DONE" && isAssistantEntry && detectSemanticCompletion(entryText)) { - log("[ralph-loop] WARNING: Semantic completion detected in transcript (agent used natural language instead of DONE)") - return true - } } catch { continue } @@ -136,13 +117,6 @@ export async function detectCompletionInSessionMessages( if (pattern.test(responseText)) { return true } - - if (SEMANTIC_DONE_FALLBACK_ENABLED && options.promise === "DONE" && detectSemanticCompletion(responseText)) { - log("[ralph-loop] WARNING: Semantic completion detected (agent used natural language instead of DONE)", { - sessionID: options.sessionID, - }) - return true - } } return false diff --git a/src/hooks/ralph-loop/completion-promise-session-negative.test.ts b/src/hooks/ralph-loop/completion-promise-session-negative.test.ts new file mode 100644 index 000000000..4805c4994 --- /dev/null +++ b/src/hooks/ralph-loop/completion-promise-session-negative.test.ts @@ -0,0 +1,104 @@ +/// +import { describe, expect, test } from "bun:test" +import { detectCompletionInSessionMessages } from "./completion-promise-detector" +import { createPluginInput } from "./completion-promise-detector-test-input" + +describe("detectCompletionInSessionMessages negative cases", () => { + describe("#given natural language completion text without explicit promise", () => { + test("#when assistant says work is complete #then should NOT detect completion", async () => { + // #given + const messages = [ + { + info: { role: "assistant" }, + parts: [{ type: "text", text: "The task is complete. All work has been finished." }], + }, + ] + const ctx = createPluginInput(messages) + + // #when + const detected = await detectCompletionInSessionMessages(ctx, { + sessionID: "session-natural-language", + promise: "DONE", + apiTimeoutMs: 1000, + directory: "/tmp", + }) + + // #then + expect(detected).toBe(false) + }) + + test("#when assistant quotes completion text while still working #then should NOT detect completion", async () => { + // #given + const messages = [ + { + info: { role: "assistant" }, + parts: [{ type: "text", text: 'The user wrote: "the task is complete". I am still working.' }], + }, + ] + const ctx = createPluginInput(messages) + + // #when + const detected = await detectCompletionInSessionMessages(ctx, { + sessionID: "session-quoted-language", + promise: "DONE", + apiTimeoutMs: 1000, + directory: "/tmp", + }) + + // #then + expect(detected).toBe(false) + }) + }) + + describe("#given promise appears outside assistant text parts", () => { + test("#when VERIFIED appears only in tool_result part #then should NOT detect completion", async () => { + // #given + const messages = [ + { + info: { role: "assistant" }, + parts: [ + { type: "tool_result", text: 'Task completed.\n\nAgent: oracle\n\nVERIFIED' }, + { type: "text", text: "Oracle verified the task." }, + ], + }, + ] + const ctx = createPluginInput(messages) + + // #when + const detected = await detectCompletionInSessionMessages(ctx, { + sessionID: "session-verified-tool-result", + promise: "VERIFIED", + apiTimeoutMs: 1000, + directory: "/tmp", + }) + + // #then + expect(detected).toBe(false) + }) + + test("#when DONE appears only in tool_result part #then should NOT detect completion", async () => { + // #given + const messages = [ + { + info: { role: "assistant" }, + parts: [ + { type: "tool_result", text: "Background task output DONE" }, + { type: "text", text: "Task completed successfully." }, + ], + }, + ] + const ctx = createPluginInput(messages) + + // #when + const detected = await detectCompletionInSessionMessages(ctx, { + sessionID: "session-done-tool-result", + promise: "DONE", + apiTimeoutMs: 1000, + directory: "/tmp", + }) + + // #then + expect(detected).toBe(false) + }) + }) +}) diff --git a/src/hooks/ralph-loop/completion-promise-transcript-detector.test.ts b/src/hooks/ralph-loop/completion-promise-transcript-detector.test.ts new file mode 100644 index 000000000..0d2509b57 --- /dev/null +++ b/src/hooks/ralph-loop/completion-promise-transcript-detector.test.ts @@ -0,0 +1,82 @@ +/// +import { afterEach, describe, expect, test } from "bun:test" +import { mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { join } from "node:path" +import { tmpdir } from "node:os" +import { detectCompletionInTranscript } from "./completion-promise-detector" + +const temporaryDirectories: string[] = [] + +function createTranscriptFile(lines: string[]): string { + const directoryPath = mkdtempSync(join(tmpdir(), "ralph-loop-transcript-")) + temporaryDirectories.push(directoryPath) + const transcriptPath = join(directoryPath, "session.jsonl") + writeFileSync(transcriptPath, `${lines.join("\n")}\n`) + return transcriptPath +} + +afterEach(() => { + for (const directoryPath of temporaryDirectories.splice(0)) { + rmSync(directoryPath, { force: true, recursive: true }) + } +}) + +describe("detectCompletionInTranscript", () => { + describe("#given transcript entries after loop start", () => { + test("#when assistant content includes explicit DONE promise #then should detect completion", () => { + // #given + const transcriptPath = createTranscriptFile([ + JSON.stringify({ type: "assistant", timestamp: "2026-03-28T10:00:00.000Z", content: "Still working" }), + JSON.stringify({ type: "assistant", timestamp: "2026-03-28T10:01:00.000Z", content: "Finished DONE" }), + ]) + + // #when + const detected = detectCompletionInTranscript(transcriptPath, "DONE", "2026-03-28T09:59:59.000Z") + + // #then + expect(detected).toBe(true) + }) + + test("#when explicit DONE appears only before startedAt #then should NOT detect completion", () => { + // #given + const transcriptPath = createTranscriptFile([ + JSON.stringify({ type: "assistant", timestamp: "2026-03-28T10:00:00.000Z", content: "Finished DONE" }), + JSON.stringify({ type: "assistant", timestamp: "2026-03-28T10:01:00.000Z", content: "Working on the new task" }), + ]) + + // #when + const detected = detectCompletionInTranscript(transcriptPath, "DONE", "2026-03-28T10:00:30.000Z") + + // #then + expect(detected).toBe(false) + }) + }) + + describe("#given transcript content without explicit assistant promise text", () => { + test("#when assistant uses only natural language completion text #then should NOT detect completion", () => { + // #given + const transcriptPath = createTranscriptFile([ + JSON.stringify({ type: "assistant", timestamp: "2026-03-28T10:01:00.000Z", content: "The task is complete. All work has been finished." }), + ]) + + // #when + const detected = detectCompletionInTranscript(transcriptPath, "DONE") + + // #then + expect(detected).toBe(false) + }) + + test("#when tool output contains DONE promise without assistant entry type #then should NOT detect completion", () => { + // #given + const transcriptPath = createTranscriptFile([ + JSON.stringify({ type: "tool_result", timestamp: "2026-03-28T10:01:00.000Z", tool_output: "Background task DONE" }), + ]) + + // #when + const detected = detectCompletionInTranscript(transcriptPath, "DONE") + + // #then + expect(detected).toBe(false) + }) + }) +})