diff --git a/src/hooks/ralph-loop/completion-promise-detector-test-input.ts b/src/hooks/ralph-loop/completion-promise-detector-test-input.ts new file mode 100644 index 000000000..4a1e6c259 --- /dev/null +++ b/src/hooks/ralph-loop/completion-promise-detector-test-input.ts @@ -0,0 +1,23 @@ +/// +import type { PluginInput } from "@opencode-ai/plugin" + +export type SessionMessage = { + info?: { role?: string } + parts?: Array<{ type: string; text?: string }> +} + +export function createPluginInput(messages: SessionMessage[]): PluginInput { + const pluginInput = { + client: { session: {} } as PluginInput["client"], + project: {} as PluginInput["project"], + directory: "/tmp", + worktree: "/tmp", + serverUrl: new URL("http://localhost"), + $: {} as PluginInput["$"], + } as PluginInput + + pluginInput.client.session.messages = + (async () => ({ data: messages })) as unknown as PluginInput["client"]["session"]["messages"] + + return pluginInput +} diff --git a/src/hooks/ralph-loop/completion-promise-detector.test.ts b/src/hooks/ralph-loop/completion-promise-detector.test.ts index 0e87bf94e..317883a6c 100644 --- a/src/hooks/ralph-loop/completion-promise-detector.test.ts +++ b/src/hooks/ralph-loop/completion-promise-detector.test.ts @@ -1,34 +1,13 @@ /// import { describe, expect, test } from "bun:test" -import type { PluginInput } from "@opencode-ai/plugin" -import { detectCompletionInSessionMessages, detectSemanticCompletion } from "./completion-promise-detector" - -type SessionMessage = { - info?: { role?: string } - parts?: Array<{ type: string; text?: string }> -} - -function createPluginInput(messages: SessionMessage[]): PluginInput { - const pluginInput = { - client: { session: {} } as PluginInput["client"], - project: {} as PluginInput["project"], - directory: "/tmp", - worktree: "/tmp", - serverUrl: new URL("http://localhost"), - $: {} as PluginInput["$"], - } as PluginInput - - pluginInput.client.session.messages = - (async () => ({ data: messages })) as unknown as PluginInput["client"]["session"]["messages"] - - return pluginInput -} +import { detectCompletionInSessionMessages } from "./completion-promise-detector" +import { createPluginInput } from "./completion-promise-detector-test-input" describe("detectCompletionInSessionMessages", () => { describe("#given session with prior DONE and new messages", () => { test("#when sinceMessageIndex excludes prior DONE #then should NOT detect completion", async () => { // #given - const messages: SessionMessage[] = [ + const messages = [ { info: { role: "assistant" }, parts: [{ type: "text", text: "Old completion DONE" }], @@ -55,7 +34,7 @@ describe("detectCompletionInSessionMessages", () => { test("#when sinceMessageIndex includes current DONE #then should detect completion", async () => { // #given - const messages: SessionMessage[] = [ + const messages = [ { info: { role: "assistant" }, parts: [{ type: "text", text: "Old completion DONE" }], @@ -84,7 +63,7 @@ describe("detectCompletionInSessionMessages", () => { describe("#given no sinceMessageIndex (backward compat)", () => { test("#then should scan all messages", async () => { // #given - const messages: SessionMessage[] = [ + const messages = [ { info: { role: "assistant" }, parts: [{ type: "text", text: "Old completion DONE" }], @@ -111,7 +90,7 @@ describe("detectCompletionInSessionMessages", () => { describe("#given promise appears in tool_result part (not text part)", () => { test("#when Oracle returns VERIFIED via task() tool_result #then should NOT detect completion", async () => { - const messages: SessionMessage[] = [ + const messages = [ { info: { role: "assistant" }, parts: [ @@ -141,7 +120,7 @@ describe("detectCompletionInSessionMessages", () => { }) test("#when DONE appears only in tool_result part #then should NOT detect completion", async () => { - const messages: SessionMessage[] = [ + const messages = [ { info: { role: "assistant" }, parts: [ @@ -163,7 +142,7 @@ describe("detectCompletionInSessionMessages", () => { }) test("#when promise appears in tool_use part (not tool_result) #then should NOT detect completion", async () => { - const messages: SessionMessage[] = [ + const messages = [ { info: { role: "assistant" }, parts: [ @@ -184,217 +163,4 @@ describe("detectCompletionInSessionMessages", () => { expect(detected).toBe(false) }) }) - - describe("#given semantic completion patterns", () => { - test("#when agent says 'task is complete' without explicit promise #then should NOT detect completion", async () => { - // #given - const messages: SessionMessage[] = [ - { - info: { role: "assistant" }, - parts: [{ type: "text", text: "The task is complete. All work has been finished." }], - }, - ] - const ctx = createPluginInput(messages) - - // #when - const detected = await detectCompletionInSessionMessages(ctx, { - sessionID: "session-123", - promise: "DONE", - apiTimeoutMs: 1000, - directory: "/tmp", - }) - - // #then - expect(detected).toBe(false) - }) - - test("#when agent says 'all items are done' without explicit promise #then should NOT detect completion", async () => { - // #given - const messages: SessionMessage[] = [ - { - info: { role: "assistant" }, - parts: [{ type: "text", text: "All items are done and marked as complete." }], - }, - ] - const ctx = createPluginInput(messages) - - // #when - const detected = await detectCompletionInSessionMessages(ctx, { - sessionID: "session-123", - promise: "DONE", - apiTimeoutMs: 1000, - directory: "/tmp", - }) - - // #then - expect(detected).toBe(false) - }) - - test("#when agent says 'nothing left to do' without explicit promise #then should NOT detect completion", async () => { - // #given - const messages: SessionMessage[] = [ - { - info: { role: "assistant" }, - parts: [{ type: "text", text: "There is nothing left to do. Everything is finished." }], - }, - ] - const ctx = createPluginInput(messages) - - // #when - const detected = await detectCompletionInSessionMessages(ctx, { - sessionID: "session-123", - promise: "DONE", - apiTimeoutMs: 1000, - directory: "/tmp", - }) - - // #then - expect(detected).toBe(false) - }) - - test("#when agent says 'successfully completed all' without explicit promise #then should NOT detect completion", async () => { - // #given - const messages: SessionMessage[] = [ - { - info: { role: "assistant" }, - parts: [{ type: "text", text: "I have successfully completed all the required tasks." }], - }, - ] - const ctx = createPluginInput(messages) - - // #when - const detected = await detectCompletionInSessionMessages(ctx, { - sessionID: "session-123", - promise: "DONE", - apiTimeoutMs: 1000, - directory: "/tmp", - }) - - // #then - expect(detected).toBe(false) - }) - - test("#when promise is VERIFIED #then semantic completion should NOT trigger", async () => { - // #given - const messages: SessionMessage[] = [ - { - info: { role: "assistant" }, - parts: [{ type: "text", text: "The task is complete. All work has been finished." }], - }, - ] - const ctx = createPluginInput(messages) - - // #when - const detected = await detectCompletionInSessionMessages(ctx, { - sessionID: "session-123", - promise: "VERIFIED", - apiTimeoutMs: 1000, - directory: "/tmp", - }) - - // #then - expect(detected).toBe(false) - }) - - test("#when completion text appears inside a quote #then should NOT detect completion", async () => { - // #given - const messages: SessionMessage[] = [ - { - info: { role: "assistant" }, - parts: [{ type: "text", text: 'The user wrote: "the task is complete". I am still working.' }], - }, - ] - const ctx = createPluginInput(messages) - - // #when - const detected = await detectCompletionInSessionMessages(ctx, { - sessionID: "session-quoted", - promise: "DONE", - apiTimeoutMs: 1000, - directory: "/tmp", - }) - - // #then - expect(detected).toBe(false) - }) - - test("#when tool_result says all items are complete #then should NOT detect completion", async () => { - // #given - const messages: SessionMessage[] = [ - { - info: { role: "assistant" }, - parts: [ - { type: "tool_result", text: "Background agent report: all items are complete." }, - { type: "text", text: "Still validating the final behavior." }, - ], - }, - ] - const ctx = createPluginInput(messages) - - // #when - const detected = await detectCompletionInSessionMessages(ctx, { - sessionID: "session-tool-result-semantic", - promise: "DONE", - apiTimeoutMs: 1000, - directory: "/tmp", - }) - - // #then - expect(detected).toBe(false) - }) - - test("#when assistant says complete but not actually done #then should NOT detect completion", async () => { - // #given - const messages: SessionMessage[] = [ - { - info: { role: "assistant" }, - parts: [{ type: "text", text: "The implementation looks complete, but I still need to run the tests." }], - }, - ] - const ctx = createPluginInput(messages) - - // #when - const detected = await detectCompletionInSessionMessages(ctx, { - sessionID: "session-not-actually-done", - promise: "DONE", - apiTimeoutMs: 1000, - directory: "/tmp", - }) - - // #then - expect(detected).toBe(false) - }) - }) -}) - -describe("detectSemanticCompletion", () => { - describe("#given semantic completion patterns", () => { - test("#when text contains 'task is complete' #then should return true", () => { - expect(detectSemanticCompletion("The task is complete.")).toBe(true) - }) - - test("#when text contains 'all items are done' #then should return true", () => { - expect(detectSemanticCompletion("All items are done.")).toBe(true) - }) - - test("#when text contains 'nothing left to do' #then should return true", () => { - expect(detectSemanticCompletion("There is nothing left to do.")).toBe(true) - }) - - test("#when text contains 'successfully completed all' #then should return true", () => { - expect(detectSemanticCompletion("Successfully completed all tasks.")).toBe(true) - }) - - test("#when text contains 'everything is finished' #then should return true", () => { - expect(detectSemanticCompletion("Everything is finished.")).toBe(true) - }) - - test("#when text does NOT contain completion patterns #then should return false", () => { - expect(detectSemanticCompletion("Working on the next task.")).toBe(false) - }) - - test("#when text is empty #then should return false", () => { - expect(detectSemanticCompletion("")).toBe(false) - }) - }) }) diff --git a/src/hooks/ralph-loop/completion-promise-session-negative.test.ts b/src/hooks/ralph-loop/completion-promise-session-negative.test.ts new file mode 100644 index 000000000..4805c4994 --- /dev/null +++ b/src/hooks/ralph-loop/completion-promise-session-negative.test.ts @@ -0,0 +1,104 @@ +/// +import { describe, expect, test } from "bun:test" +import { detectCompletionInSessionMessages } from "./completion-promise-detector" +import { createPluginInput } from "./completion-promise-detector-test-input" + +describe("detectCompletionInSessionMessages negative cases", () => { + describe("#given natural language completion text without explicit promise", () => { + test("#when assistant says work is complete #then should NOT detect completion", async () => { + // #given + const messages = [ + { + info: { role: "assistant" }, + parts: [{ type: "text", text: "The task is complete. All work has been finished." }], + }, + ] + const ctx = createPluginInput(messages) + + // #when + const detected = await detectCompletionInSessionMessages(ctx, { + sessionID: "session-natural-language", + promise: "DONE", + apiTimeoutMs: 1000, + directory: "/tmp", + }) + + // #then + expect(detected).toBe(false) + }) + + test("#when assistant quotes completion text while still working #then should NOT detect completion", async () => { + // #given + const messages = [ + { + info: { role: "assistant" }, + parts: [{ type: "text", text: 'The user wrote: "the task is complete". I am still working.' }], + }, + ] + const ctx = createPluginInput(messages) + + // #when + const detected = await detectCompletionInSessionMessages(ctx, { + sessionID: "session-quoted-language", + promise: "DONE", + apiTimeoutMs: 1000, + directory: "/tmp", + }) + + // #then + expect(detected).toBe(false) + }) + }) + + describe("#given promise appears outside assistant text parts", () => { + test("#when VERIFIED appears only in tool_result part #then should NOT detect completion", async () => { + // #given + const messages = [ + { + info: { role: "assistant" }, + parts: [ + { type: "tool_result", text: 'Task completed.\n\nAgent: oracle\n\nVERIFIED' }, + { type: "text", text: "Oracle verified the task." }, + ], + }, + ] + const ctx = createPluginInput(messages) + + // #when + const detected = await detectCompletionInSessionMessages(ctx, { + sessionID: "session-verified-tool-result", + promise: "VERIFIED", + apiTimeoutMs: 1000, + directory: "/tmp", + }) + + // #then + expect(detected).toBe(false) + }) + + test("#when DONE appears only in tool_result part #then should NOT detect completion", async () => { + // #given + const messages = [ + { + info: { role: "assistant" }, + parts: [ + { type: "tool_result", text: "Background task output DONE" }, + { type: "text", text: "Task completed successfully." }, + ], + }, + ] + const ctx = createPluginInput(messages) + + // #when + const detected = await detectCompletionInSessionMessages(ctx, { + sessionID: "session-done-tool-result", + promise: "DONE", + apiTimeoutMs: 1000, + directory: "/tmp", + }) + + // #then + expect(detected).toBe(false) + }) + }) +})