ralph-loop: detect oracle VERIFIED tool results
This commit is contained in:
@@ -2,6 +2,7 @@ import type { PluginInput } from "@opencode-ai/plugin"
|
||||
import { existsSync, readFileSync } from "node:fs"
|
||||
import { log } from "../../shared/logger"
|
||||
import { HOOK_NAME } from "./constants"
|
||||
import { ULTRAWORK_VERIFICATION_PROMISE } from "./constants"
|
||||
import { withTimeout } from "./with-timeout"
|
||||
|
||||
interface OpenCodeSessionMessage {
|
||||
@@ -16,6 +17,8 @@ interface TranscriptEntry {
|
||||
tool_output?: { output?: string } | string
|
||||
}
|
||||
|
||||
const ORACLE_AGENT_PATTERN = /Agent:\s*oracle/i
|
||||
|
||||
function extractTranscriptEntryText(entry: TranscriptEntry): string {
|
||||
if (typeof entry.content === "string") return entry.content
|
||||
if (typeof entry.tool_output === "string") return entry.tool_output
|
||||
@@ -31,6 +34,22 @@ function buildPromisePattern(promise: string): RegExp {
|
||||
return new RegExp(`<promise>\\s*${escapeRegex(promise)}\\s*</promise>`, "is")
|
||||
}
|
||||
|
||||
function shouldInspectTranscriptEntry(
|
||||
entry: TranscriptEntry,
|
||||
promise: string,
|
||||
entryText: string,
|
||||
): boolean {
|
||||
if (entry.type === "assistant" || entry.type === "text") {
|
||||
return true
|
||||
}
|
||||
|
||||
if (entry.type !== "tool_result") {
|
||||
return false
|
||||
}
|
||||
|
||||
return promise === ULTRAWORK_VERIFICATION_PROMISE && ORACLE_AGENT_PATTERN.test(entryText)
|
||||
}
|
||||
|
||||
export function detectCompletionInTranscript(
|
||||
transcriptPath: string | undefined,
|
||||
promise: string,
|
||||
@@ -49,10 +68,10 @@ export function detectCompletionInTranscript(
|
||||
try {
|
||||
const entry = JSON.parse(line) as TranscriptEntry
|
||||
if (entry.type === "user") continue
|
||||
if (entry.type !== "assistant" && entry.type !== "text") continue
|
||||
if (startedAt && entry.timestamp && entry.timestamp < startedAt) continue
|
||||
const entryText = extractTranscriptEntryText(entry)
|
||||
if (!entryText) continue
|
||||
if (!shouldInspectTranscriptEntry(entry, promise, entryText)) continue
|
||||
if (pattern.test(entryText)) return true
|
||||
} catch {
|
||||
continue
|
||||
|
||||
@@ -78,5 +78,22 @@ describe("detectCompletionInTranscript", () => {
|
||||
// #then
|
||||
expect(detected).toBe(false)
|
||||
})
|
||||
|
||||
test("#when oracle tool output contains VERIFIED promise #then should detect verification completion", () => {
|
||||
// #given
|
||||
const transcriptPath = createTranscriptFile([
|
||||
JSON.stringify({
|
||||
type: "tool_result",
|
||||
timestamp: "2026-03-28T10:01:00.000Z",
|
||||
tool_output: "Task completed.\n\nAgent: oracle\n\n<promise>VERIFIED</promise>\n\n<task_metadata>\nsession_id: ses_oracle_123\n</task_metadata>",
|
||||
}),
|
||||
])
|
||||
|
||||
// #when
|
||||
const detected = detectCompletionInTranscript(transcriptPath, "VERIFIED")
|
||||
|
||||
// #then
|
||||
expect(detected).toBe(true)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -135,6 +135,36 @@ describe("ulw-loop verification", () => {
|
||||
expect(toastCalls.some((toast) => toast.title === "ULTRAWORK LOOP COMPLETE!")).toBe(true)
|
||||
})
|
||||
|
||||
test("#given ulw loop is awaiting verification #when oracle transcript stores VERIFIED inside tool_result #then loop completes", async () => {
|
||||
const hook = createRalphLoopHook(createMockPluginInput(), {
|
||||
getTranscriptPath: (sessionID) => sessionID === "ses-oracle" ? oracleTranscriptPath : parentTranscriptPath,
|
||||
})
|
||||
hook.startLoop("session-123", "Build API", { ultrawork: true })
|
||||
writeFileSync(
|
||||
parentTranscriptPath,
|
||||
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: "done <promise>DONE</promise>" })}\n`,
|
||||
)
|
||||
|
||||
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
|
||||
writeState(testDir, {
|
||||
...hook.getState()!,
|
||||
verification_session_id: "ses-oracle",
|
||||
})
|
||||
writeFileSync(
|
||||
oracleTranscriptPath,
|
||||
`${JSON.stringify({
|
||||
type: "tool_result",
|
||||
timestamp: new Date().toISOString(),
|
||||
tool_output: `Task completed.\n\nAgent: oracle\n\n<promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>\n\n<task_metadata>\nsession_id: ses-oracle\n</task_metadata>`,
|
||||
})}\n`,
|
||||
)
|
||||
|
||||
await hook.event({ event: { type: "session.idle", properties: { sessionID: "ses-oracle" } } })
|
||||
|
||||
expect(hook.getState()).toBeNull()
|
||||
expect(toastCalls.some((toast) => toast.title === "ULTRAWORK LOOP COMPLETE!")).toBe(true)
|
||||
})
|
||||
|
||||
test("#given ulw loop is awaiting verification without oracle session #when parent idles again #then loop continues until oracle verifies", async () => {
|
||||
const hook = createRalphLoopHook(createMockPluginInput(), {
|
||||
getTranscriptPath: (sessionID) => sessionID === "ses-oracle" ? oracleTranscriptPath : parentTranscriptPath,
|
||||
|
||||
Reference in New Issue
Block a user