ralph-loop: detect oracle VERIFIED tool results

This commit is contained in:
YeonGyu-Kim
2026-04-04 19:03:57 +09:00
parent f7d2ff75a7
commit 39f6ab2b44
3 changed files with 67 additions and 1 deletions
@@ -2,6 +2,7 @@ import type { PluginInput } from "@opencode-ai/plugin"
import { existsSync, readFileSync } from "node:fs"
import { log } from "../../shared/logger"
import { HOOK_NAME } from "./constants"
import { ULTRAWORK_VERIFICATION_PROMISE } from "./constants"
import { withTimeout } from "./with-timeout"
interface OpenCodeSessionMessage {
@@ -16,6 +17,8 @@ interface TranscriptEntry {
tool_output?: { output?: string } | string
}
const ORACLE_AGENT_PATTERN = /Agent:\s*oracle/i
function extractTranscriptEntryText(entry: TranscriptEntry): string {
if (typeof entry.content === "string") return entry.content
if (typeof entry.tool_output === "string") return entry.tool_output
@@ -31,6 +34,22 @@ function buildPromisePattern(promise: string): RegExp {
return new RegExp(`<promise>\\s*${escapeRegex(promise)}\\s*</promise>`, "is")
}
function shouldInspectTranscriptEntry(
entry: TranscriptEntry,
promise: string,
entryText: string,
): boolean {
if (entry.type === "assistant" || entry.type === "text") {
return true
}
if (entry.type !== "tool_result") {
return false
}
return promise === ULTRAWORK_VERIFICATION_PROMISE && ORACLE_AGENT_PATTERN.test(entryText)
}
export function detectCompletionInTranscript(
transcriptPath: string | undefined,
promise: string,
@@ -49,10 +68,10 @@ export function detectCompletionInTranscript(
try {
const entry = JSON.parse(line) as TranscriptEntry
if (entry.type === "user") continue
if (entry.type !== "assistant" && entry.type !== "text") continue
if (startedAt && entry.timestamp && entry.timestamp < startedAt) continue
const entryText = extractTranscriptEntryText(entry)
if (!entryText) continue
if (!shouldInspectTranscriptEntry(entry, promise, entryText)) continue
if (pattern.test(entryText)) return true
} catch {
continue
@@ -78,5 +78,22 @@ describe("detectCompletionInTranscript", () => {
// #then
expect(detected).toBe(false)
})
test("#when oracle tool output contains VERIFIED promise #then should detect verification completion", () => {
// #given
const transcriptPath = createTranscriptFile([
JSON.stringify({
type: "tool_result",
timestamp: "2026-03-28T10:01:00.000Z",
tool_output: "Task completed.\n\nAgent: oracle\n\n<promise>VERIFIED</promise>\n\n<task_metadata>\nsession_id: ses_oracle_123\n</task_metadata>",
}),
])
// #when
const detected = detectCompletionInTranscript(transcriptPath, "VERIFIED")
// #then
expect(detected).toBe(true)
})
})
})
@@ -135,6 +135,36 @@ describe("ulw-loop verification", () => {
expect(toastCalls.some((toast) => toast.title === "ULTRAWORK LOOP COMPLETE!")).toBe(true)
})
test("#given ulw loop is awaiting verification #when oracle transcript stores VERIFIED inside tool_result #then loop completes", async () => {
const hook = createRalphLoopHook(createMockPluginInput(), {
getTranscriptPath: (sessionID) => sessionID === "ses-oracle" ? oracleTranscriptPath : parentTranscriptPath,
})
hook.startLoop("session-123", "Build API", { ultrawork: true })
writeFileSync(
parentTranscriptPath,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: "done <promise>DONE</promise>" })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
writeState(testDir, {
...hook.getState()!,
verification_session_id: "ses-oracle",
})
writeFileSync(
oracleTranscriptPath,
`${JSON.stringify({
type: "tool_result",
timestamp: new Date().toISOString(),
tool_output: `Task completed.\n\nAgent: oracle\n\n<promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>\n\n<task_metadata>\nsession_id: ses-oracle\n</task_metadata>`,
})}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "ses-oracle" } } })
expect(hook.getState()).toBeNull()
expect(toastCalls.some((toast) => toast.title === "ULTRAWORK LOOP COMPLETE!")).toBe(true)
})
test("#given ulw loop is awaiting verification without oracle session #when parent idles again #then loop continues until oracle verifies", async () => {
const hook = createRalphLoopHook(createMockPluginInput(), {
getTranscriptPath: (sessionID) => sessionID === "ses-oracle" ? oracleTranscriptPath : parentTranscriptPath,