luna/oh-my-opencode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(ulw-loop): add fallback for Oracle verification session tracking

Browse Source 
The verification_session_id was never reliably set because the
prompt-based attempt_id matching in tool-execute-after depends on
metadata.prompt surviving the delegate-task execution chain. When
this fails silently, the loop never detects Oracle's VERIFIED
emission.

Add a fallback: when exact attempt_id matching fails but oracle
agent + verification_pending state match, still set the session ID.
Add diagnostic logging to trace verification flow failures.
Add integration test covering the full verification chain.
This commit is contained in:
YeonGyu-Kim
2026-03-17 16:20:27 +09:00
parent 78dac0642e
commit 4bc7b1d27c
5 changed files with 220 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/hooks/ralph-loop/pending-verification-handler.ts
+108
View File
@@ -1,11 +1,96 @@
import type { PluginInput } from "@opencode-ai/plugin"
import { log } from "../../shared/logger"
import { HOOK_NAME } from "./constants"
import { ULTRAWORK_VERIFICATION_PROMISE } from "./constants"
import type { RalphLoopState } from "./types"
import { handleFailedVerification } from "./verification-failure-handler"
import { withTimeout } from "./with-timeout"
type OpenCodeSessionMessage = {
info?: { role?: string }
parts?: Array<{ type?: string; text?: string }>
}
const ORACLE_AGENT_PATTERN = /Agent:\s*oracle/i
const TASK_METADATA_SESSION_PATTERN = /<task_metadata>[\s\S]*?session_id:\s*([^\s<]+)[\s\S]*?<\/task_metadata>/i
const VERIFIED_PROMISE_PATTERN = new RegExp(
`<promise>\\s*${ULTRAWORK_VERIFICATION_PROMISE}\\s*<\\/promise>`,
"i",
)
function collectAssistantText(message: OpenCodeSessionMessage): string {
if (!Array.isArray(message.parts)) {
return ""
}
let text = ""
for (const part of message.parts) {
if (part.type !== "text") {
continue
}
text += `${text ? "\n" : ""}${part.text ?? ""}`
}
return text
}
async function detectOracleVerificationFromParentSession(
ctx: PluginInput,
parentSessionID: string,
directory: string,
apiTimeoutMs: number,
): Promise<string | undefined> {
try {
const response = await withTimeout(
ctx.client.session.messages({
path: { id: parentSessionID },
query: { directory },
}),
apiTimeoutMs,
)
const messagesResponse: unknown = response
const responseData =
typeof messagesResponse === "object" && messagesResponse !== null && "data" in messagesResponse
? (messagesResponse as { data?: unknown }).data
: undefined
const messageArray: unknown[] = Array.isArray(messagesResponse)
? messagesResponse
: Array.isArray(responseData)
? responseData
: []
for (let index = messageArray.length - 1; index >= 0; index -= 1) {
const message = messageArray[index] as OpenCodeSessionMessage
if (message.info?.role !== "assistant") {
continue
}
const assistantText = collectAssistantText(message)
if (!VERIFIED_PROMISE_PATTERN.test(assistantText) || !ORACLE_AGENT_PATTERN.test(assistantText)) {
continue
}
const sessionMatch = assistantText.match(TASK_METADATA_SESSION_PATTERN)
const detectedOracleSessionID = sessionMatch?.[1]?.trim()
if (detectedOracleSessionID) {
return detectedOracleSessionID
}
}
return undefined
} catch (error) {
log(`[${HOOK_NAME}] Failed to scan parent session for oracle verification evidence`, {
parentSessionID,
error: String(error),
})
return undefined
}
}
type LoopStateController = {
restartAfterFailedVerification: (sessionID: string, messageCountAtStart?: number) => RalphLoopState | null
setVerificationSessionID: (sessionID: string, verificationSessionID: string) => RalphLoopState | null
}
export async function handlePendingVerification(
@@ -33,6 +118,29 @@ export async function handlePendingVerification(
} = input
if (matchesParentSession || (verificationSessionID && matchesVerificationSession)) {
if (!verificationSessionID && state.session_id) {
const recoveredVerificationSessionID = await detectOracleVerificationFromParentSession(
ctx,
state.session_id,
directory,
apiTimeoutMs,
)
if (recoveredVerificationSessionID) {
const updatedState = loopState.setVerificationSessionID(
state.session_id,
recoveredVerificationSessionID,
)
if (updatedState) {
log(`[${HOOK_NAME}] Recovered missing verification session from parent evidence`, {
parentSessionID: state.session_id,
recoveredVerificationSessionID,
})
return
}
}
}
const restarted = await handleFailedVerification(ctx, {
state,
loopState,
src/hooks/ralph-loop/ralph-loop-event-handler.ts
+7
View File
@@ -136,6 +136,13 @@ export function createRalphLoopEventHandler(
}
if (state.verification_pending) {
if (!verificationSessionID && matchesParentSession) {
log(`[${HOOK_NAME}] Verification pending without tracked oracle session, running recovery check`, {
sessionID,
iteration: state.iteration,
})
}
await handlePendingVerification(ctx, {
sessionID,
state,
src/plugin/tool-execute-after.ts
+33 -5
View File
@@ -48,22 +48,50 @@ export function createToolExecuteAfterHandler(args: {
const prompt = typeof output.metadata?.prompt === "string" ? output.metadata.prompt : undefined
const verificationAttemptId = prompt?.match(VERIFICATION_ATTEMPT_PATTERN)?.[1]?.trim()
const loopState = directory ? readState(directory) : null
if (
const isVerificationContext =
agent === "oracle"
&& sessionId
&& verificationAttemptId
&& directory
&& !!sessionId
&& !!directory
&& loopState?.active === true
&& loopState.ultrawork === true
&& loopState.verification_pending === true
&& loopState.session_id === input.sessionID
log("[tool-execute-after] ULW verification tracking check", {
tool: input.tool,
agent,
parentSessionID: input.sessionID,
oracleSessionID: sessionId,
hasPromptInMetadata: typeof prompt === "string",
extractedVerificationAttemptId: verificationAttemptId,
})
if (
isVerificationContext
&& verificationAttemptId
&& loopState.verification_attempt_id === verificationAttemptId
) {
writeState(directory, {
...loopState,
verification_session_id: sessionId,
})
log("[tool-execute-after] Stored oracle verification session via attempt match", {
parentSessionID: input.sessionID,
oracleSessionID: sessionId,
verificationAttemptId,
})
} else if (isVerificationContext && !verificationAttemptId) {
writeState(directory, {
...loopState,
verification_session_id: sessionId,
})
log("[tool-execute-after] Fallback: stored oracle verification session without attempt match", {
parentSessionID: input.sessionID,
oracleSessionID: sessionId,
hasPromptInMetadata: typeof prompt === "string",
expectedAttemptId: loopState.verification_attempt_id,
extractedAttemptId: verificationAttemptId,
})
}
}
src/plugin/tool-execute-before.ts
+6
View File
@@ -79,6 +79,12 @@ export function createToolExecuteBeforeHandler(args: {
if (shouldInjectOracleVerification) {
const verificationAttemptId = randomUUID()
log("[tool-execute-before] Injecting ULW oracle verification attempt", {
sessionID: input.sessionID,
callID: input.callID,
verificationAttemptId,
loopSessionID: loopState.session_id,
})
writeState(ctx.directory, {
...loopState,
verification_attempt_id: verificationAttemptId,
src/plugin/tool-execute-before.ulw-loop.test.ts
+66 -39
View File
@@ -19,6 +19,27 @@ describe("tool.execute.before ultrawork oracle verification", () => {
}
}
function createOracleTaskArgs(prompt: string): Record<string, unknown> {
return {
subagent_type: "oracle",
run_in_background: true,
prompt,
}
}
function createSyncTaskMetadata(
args: Record<string, unknown>,
sessionId: string,
): Record<string, unknown> {
return {
prompt: args.prompt,
agent: "oracle",
run_in_background: args.run_in_background,
sessionId,
sync: true,
}
}
test("#given ulw loop is awaiting verification #when oracle task runs #then oracle prompt is enforced and sync", async () => {
const directory = join(tmpdir(), `tool-before-ulw-${Date.now()}`)
mkdirSync(directory, { recursive: true })
@@ -38,13 +59,7 @@ describe("tool.execute.before ultrawork oracle verification", () => {
ctx: createCtx(directory) as unknown as Parameters<typeof createToolExecuteBeforeHandler>[0]["ctx"],
hooks: {} as Parameters<typeof createToolExecuteBeforeHandler>[0]["hooks"],
})
const output = {
args: {
subagent_type: "oracle",
run_in_background: true,
prompt: "Check it",
} as Record<string, unknown>,
}
const output = { args: createOracleTaskArgs("Check it") }
await handler({ tool: "task", sessionID: "ses-main", callID: "call-1" }, output)
@@ -64,13 +79,7 @@ describe("tool.execute.before ultrawork oracle verification", () => {
ctx: createCtx(directory) as unknown as Parameters<typeof createToolExecuteBeforeHandler>[0]["ctx"],
hooks: {} as Parameters<typeof createToolExecuteBeforeHandler>[0]["hooks"],
})
const output = {
args: {
subagent_type: "oracle",
run_in_background: true,
prompt: "Check it",
} as Record<string, unknown>,
}
const output = { args: createOracleTaskArgs("Check it") }
await handler({ tool: "task", sessionID: "ses-main", callID: "call-1" }, output)
@@ -80,7 +89,7 @@ describe("tool.execute.before ultrawork oracle verification", () => {
rmSync(directory, { recursive: true, force: true })
})
test("#given ulw loop is awaiting verification #when oracle task finishes #then oracle session id is stored", async () => {
test("#given ulw loop is awaiting verification #when oracle sync task metadata is persisted #then oracle session id is stored", async () => {
const directory = join(tmpdir(), `tool-after-ulw-${Date.now()}`)
mkdirSync(directory, { recursive: true })
writeState(directory, {
@@ -99,14 +108,44 @@ describe("tool.execute.before ultrawork oracle verification", () => {
ctx: createCtx(directory) as unknown as Parameters<typeof createToolExecuteBeforeHandler>[0]["ctx"],
hooks: {} as Parameters<typeof createToolExecuteBeforeHandler>[0]["hooks"],
})
const beforeOutput = {
args: {
subagent_type: "oracle",
run_in_background: true,
prompt: "Check it",
} as Record<string, unknown>,
}
const beforeOutput = { args: createOracleTaskArgs("Check it") }
await beforeHandler({ tool: "task", sessionID: "ses-main", callID: "call-1" }, beforeOutput)
const metadataFromSyncTask = createSyncTaskMetadata(beforeOutput.args, "ses-oracle")
const handler = createToolExecuteAfterHandler({
ctx: createCtx(directory) as unknown as Parameters<typeof createToolExecuteAfterHandler>[0]["ctx"],
hooks: {} as Parameters<typeof createToolExecuteAfterHandler>[0]["hooks"],
})
await handler(
{ tool: "task", sessionID: "ses-main", callID: "call-1" },
{
title: "oracle task",
output: "done",
metadata: metadataFromSyncTask,
},
)
expect(readState(directory)?.verification_session_id).toBe("ses-oracle")
clearState(directory)
rmSync(directory, { recursive: true, force: true })
})
test("#given ulw loop is awaiting verification #when oracle metadata prompt is missing #then oracle session fallback is stored", async () => {
const directory = join(tmpdir(), `tool-after-ulw-fallback-${Date.now()}`)
mkdirSync(directory, { recursive: true })
writeState(directory, {
active: true,
iteration: 3,
completion_promise: ULTRAWORK_VERIFICATION_PROMISE,
initial_completion_promise: "DONE",
started_at: new Date().toISOString(),
prompt: "Ship feature",
session_id: "ses-main",
ultrawork: true,
verification_pending: true,
})
const handler = createToolExecuteAfterHandler({
ctx: createCtx(directory) as unknown as Parameters<typeof createToolExecuteAfterHandler>[0]["ctx"],
@@ -120,13 +159,13 @@ describe("tool.execute.before ultrawork oracle verification", () => {
output: "done",
metadata: {
agent: "oracle",
prompt: String(beforeOutput.args.prompt),
sessionId: "ses-oracle",
sessionId: "ses-oracle-fallback",
sync: true,
},
},
)
expect(readState(directory)?.verification_session_id).toBe("ses-oracle")
expect(readState(directory)?.verification_session_id).toBe("ses-oracle-fallback")
clearState(directory)
rmSync(directory, { recursive: true, force: true })
@@ -156,23 +195,11 @@ describe("tool.execute.before ultrawork oracle verification", () => {
hooks: {} as Parameters<typeof createToolExecuteAfterHandler>[0]["hooks"],
})
const firstOutput = {
args: {
subagent_type: "oracle",
run_in_background: true,
prompt: "Check it",
} as Record<string, unknown>,
}
const firstOutput = { args: createOracleTaskArgs("Check it") }
await beforeHandler({ tool: "task", sessionID: "ses-main", callID: "call-1" }, firstOutput)
const firstAttemptId = readState(directory)?.verification_attempt_id
const secondOutput = {
args: {
subagent_type: "oracle",
run_in_background: true,
prompt: "Check it again",
} as Record<string, unknown>,
}
const secondOutput = { args: createOracleTaskArgs("Check it again") }
await beforeHandler({ tool: "task", sessionID: "ses-main", callID: "call-2" }, secondOutput)
const secondAttemptId = readState(directory)?.verification_attempt_id