fix(ralph-loop): harden Oracle VERIFIED detection
Replace fragile regex text matching with structured detection for Oracle verification evidence. - Add oracle-verification-detector.ts with parseOracleVerificationEvidence() - Use structured parsing instead of multiple regex patterns - Add comprehensive test coverage for edge cases - Update completion-promise-detector.ts to use isOracleVerified() - Update pending-verification-handler.ts to use structured extraction Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
YeonGyu-Kim
@@ -3,6 +3,7 @@ import { existsSync, readFileSync } from "node:fs"
|
||||
import { log } from "../../shared/logger"
|
||||
import { HOOK_NAME } from "./constants"
|
||||
import { ULTRAWORK_VERIFICATION_PROMISE } from "./constants"
|
||||
import { isOracleVerified } from "./oracle-verification-detector"
|
||||
import { withTimeout } from "./with-timeout"
|
||||
|
||||
interface OpenCodeSessionMessage {
|
||||
@@ -17,8 +18,6 @@ interface TranscriptEntry {
|
||||
tool_output?: { output?: string } | string
|
||||
}
|
||||
|
||||
const ORACLE_AGENT_PATTERN = /Agent:\s*oracle/i
|
||||
|
||||
function extractTranscriptEntryText(entry: TranscriptEntry): string {
|
||||
if (typeof entry.content === "string") return entry.content
|
||||
if (typeof entry.tool_output === "string") return entry.tool_output
|
||||
@@ -47,7 +46,7 @@ function shouldInspectSessionMessagePart(
|
||||
return false
|
||||
}
|
||||
|
||||
return promise === ULTRAWORK_VERIFICATION_PROMISE && ORACLE_AGENT_PATTERN.test(partText)
|
||||
return promise === ULTRAWORK_VERIFICATION_PROMISE && isOracleVerified(partText)
|
||||
}
|
||||
|
||||
function shouldInspectTranscriptEntry(
|
||||
@@ -63,7 +62,7 @@ function shouldInspectTranscriptEntry(
|
||||
return false
|
||||
}
|
||||
|
||||
return promise === ULTRAWORK_VERIFICATION_PROMISE && ORACLE_AGENT_PATTERN.test(entryText)
|
||||
return promise === ULTRAWORK_VERIFICATION_PROMISE && isOracleVerified(entryText)
|
||||
}
|
||||
|
||||
export function detectCompletionInTranscript(
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
/// <reference types="bun-types" />
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import {
|
||||
extractOracleSessionID,
|
||||
isOracleVerified,
|
||||
parseOracleVerificationEvidence,
|
||||
} from "./oracle-verification-detector"
|
||||
import { ULTRAWORK_VERIFICATION_PROMISE } from "./constants"
|
||||
|
||||
describe("parseOracleVerificationEvidence", () => {
|
||||
test("#given valid oracle verification text #then should parse all fields", () => {
|
||||
// #given
|
||||
const text = `Task completed.
|
||||
|
||||
Agent: oracle
|
||||
|
||||
<promise>VERIFIED</promise>
|
||||
|
||||
<task_metadata>
|
||||
session_id: ses_oracle_123
|
||||
</task_metadata>`
|
||||
|
||||
// #when
|
||||
const evidence = parseOracleVerificationEvidence(text)
|
||||
|
||||
// #then
|
||||
expect(evidence).toBeDefined()
|
||||
expect(evidence?.agent).toBe("oracle")
|
||||
expect(evidence?.promise).toBe("VERIFIED")
|
||||
expect(evidence?.sessionID).toBe("ses_oracle_123")
|
||||
})
|
||||
|
||||
test("#given text without agent line #then should return undefined", () => {
|
||||
// #given
|
||||
const text = `<promise>VERIFIED</promise>`
|
||||
|
||||
// #when
|
||||
const evidence = parseOracleVerificationEvidence(text)
|
||||
|
||||
// #then
|
||||
expect(evidence).toBeUndefined()
|
||||
})
|
||||
|
||||
test("#given text without promise tag #then should return undefined", () => {
|
||||
// #given
|
||||
const text = `Agent: oracle`
|
||||
|
||||
// #when
|
||||
const evidence = parseOracleVerificationEvidence(text)
|
||||
|
||||
// #then
|
||||
expect(evidence).toBeUndefined()
|
||||
})
|
||||
|
||||
test("#given text with empty agent #then should return undefined", () => {
|
||||
// #given
|
||||
const text = `Agent:
|
||||
|
||||
<promise>VERIFIED</promise>`
|
||||
|
||||
// #when
|
||||
const evidence = parseOracleVerificationEvidence(text)
|
||||
|
||||
// #then
|
||||
expect(evidence).toBeUndefined()
|
||||
})
|
||||
|
||||
test("#given text with empty promise #then should return undefined", () => {
|
||||
// #given
|
||||
const text = `Agent: oracle
|
||||
|
||||
<promise> </promise>`
|
||||
|
||||
// #when
|
||||
const evidence = parseOracleVerificationEvidence(text)
|
||||
|
||||
// #then
|
||||
expect(evidence).toBeUndefined()
|
||||
})
|
||||
|
||||
test("#given text without metadata #then should parse agent and promise only", () => {
|
||||
// #given
|
||||
const text = `Agent: oracle
|
||||
|
||||
<promise>VERIFIED</promise>`
|
||||
|
||||
// #when
|
||||
const evidence = parseOracleVerificationEvidence(text)
|
||||
|
||||
// #then
|
||||
expect(evidence).toBeDefined()
|
||||
expect(evidence?.agent).toBe("oracle")
|
||||
expect(evidence?.promise).toBe("VERIFIED")
|
||||
expect(evidence?.sessionID).toBeUndefined()
|
||||
})
|
||||
|
||||
test("#given text with metadata but no session_id #then should parse agent and promise only", () => {
|
||||
// #given
|
||||
const text = `Agent: oracle
|
||||
|
||||
<promise>VERIFIED</promise>
|
||||
|
||||
<task_metadata>
|
||||
other_field: value
|
||||
</task_metadata>`
|
||||
|
||||
// #when
|
||||
const evidence = parseOracleVerificationEvidence(text)
|
||||
|
||||
// #then
|
||||
expect(evidence).toBeDefined()
|
||||
expect(evidence?.agent).toBe("oracle")
|
||||
expect(evidence?.promise).toBe("VERIFIED")
|
||||
expect(evidence?.sessionID).toBeUndefined()
|
||||
})
|
||||
|
||||
test("#given empty text #then should return undefined", () => {
|
||||
// #given
|
||||
const text = ""
|
||||
|
||||
// #when
|
||||
const evidence = parseOracleVerificationEvidence(text)
|
||||
|
||||
// #then
|
||||
expect(evidence).toBeUndefined()
|
||||
})
|
||||
|
||||
test("#given whitespace-only text #then should return undefined", () => {
|
||||
// #given
|
||||
const text = " \n\t "
|
||||
|
||||
// #when
|
||||
const evidence = parseOracleVerificationEvidence(text)
|
||||
|
||||
// #then
|
||||
expect(evidence).toBeUndefined()
|
||||
})
|
||||
|
||||
test("#given agent with different casing #then should preserve original case", () => {
|
||||
// #given
|
||||
const text = `Agent: ORACLE
|
||||
|
||||
<promise>VERIFIED</promise>`
|
||||
|
||||
// #when
|
||||
const evidence = parseOracleVerificationEvidence(text)
|
||||
|
||||
// #then
|
||||
expect(evidence).toBeDefined()
|
||||
expect(evidence?.agent).toBe("ORACLE")
|
||||
})
|
||||
})
|
||||
|
||||
describe("isOracleVerified", () => {
|
||||
test("#given valid oracle verification #then should return true", () => {
|
||||
// #given
|
||||
const text = `Agent: oracle
|
||||
|
||||
<promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>`
|
||||
|
||||
// #when
|
||||
const result = isOracleVerified(text)
|
||||
|
||||
// #then
|
||||
expect(result).toBe(true)
|
||||
})
|
||||
|
||||
test("#given non-oracle agent #then should return false", () => {
|
||||
// #given
|
||||
const text = `Agent: sisyphus
|
||||
|
||||
<promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>`
|
||||
|
||||
// #when
|
||||
const result = isOracleVerified(text)
|
||||
|
||||
// #then
|
||||
expect(result).toBe(false)
|
||||
})
|
||||
|
||||
test("#given wrong promise #then should return false", () => {
|
||||
// #given
|
||||
const text = `Agent: oracle
|
||||
|
||||
<promise>DONE</promise>`
|
||||
|
||||
// #when
|
||||
const result = isOracleVerified(text)
|
||||
|
||||
// #then
|
||||
expect(result).toBe(false)
|
||||
})
|
||||
|
||||
test("#given oracle agent with different casing #then should return true", () => {
|
||||
// #given
|
||||
const text = `Agent: ORACLE
|
||||
|
||||
<promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>`
|
||||
|
||||
// #when
|
||||
const result = isOracleVerified(text)
|
||||
|
||||
// #then
|
||||
expect(result).toBe(true)
|
||||
})
|
||||
|
||||
test("#given empty text #then should return false", () => {
|
||||
// #given
|
||||
const text = ""
|
||||
|
||||
// #when
|
||||
const result = isOracleVerified(text)
|
||||
|
||||
// #then
|
||||
expect(result).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe("extractOracleSessionID", () => {
|
||||
test("#given valid oracle verification with session_id #then should return session_id", () => {
|
||||
// #given
|
||||
const text = `Agent: oracle
|
||||
|
||||
<promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>
|
||||
|
||||
<task_metadata>
|
||||
session_id: ses_oracle_123
|
||||
</task_metadata>`
|
||||
|
||||
// #when
|
||||
const sessionID = extractOracleSessionID(text)
|
||||
|
||||
// #then
|
||||
expect(sessionID).toBe("ses_oracle_123")
|
||||
})
|
||||
|
||||
test("#given valid oracle verification without session_id #then should return undefined", () => {
|
||||
// #given
|
||||
const text = `Agent: oracle
|
||||
|
||||
<promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>`
|
||||
|
||||
// #when
|
||||
const sessionID = extractOracleSessionID(text)
|
||||
|
||||
// #then
|
||||
expect(sessionID).toBeUndefined()
|
||||
})
|
||||
|
||||
test("#given non-oracle agent #then should return undefined", () => {
|
||||
// #given
|
||||
const text = `Agent: sisyphus
|
||||
|
||||
<promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>
|
||||
|
||||
<task_metadata>
|
||||
session_id: ses_sis_123
|
||||
</task_metadata>`
|
||||
|
||||
// #when
|
||||
const sessionID = extractOracleSessionID(text)
|
||||
|
||||
// #then
|
||||
expect(sessionID).toBeUndefined()
|
||||
})
|
||||
|
||||
test("#given non-oracle agent with different casing #then should return undefined", () => {
|
||||
// #given
|
||||
const text = `Agent: SISYPHUS
|
||||
|
||||
<promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>
|
||||
|
||||
<task_metadata>
|
||||
session_id: ses_sis_123
|
||||
</task_metadata>`
|
||||
|
||||
// #when
|
||||
const sessionID = extractOracleSessionID(text)
|
||||
|
||||
// #then
|
||||
expect(sessionID).toBeUndefined()
|
||||
})
|
||||
|
||||
test("#given empty text #then should return undefined", () => {
|
||||
// #given
|
||||
const text = ""
|
||||
|
||||
// #when
|
||||
const sessionID = extractOracleSessionID(text)
|
||||
|
||||
// #then
|
||||
expect(sessionID).toBeUndefined()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,70 @@
|
||||
import { ULTRAWORK_VERIFICATION_PROMISE } from "./constants"
|
||||
|
||||
export interface OracleVerificationEvidence {
|
||||
agent: string
|
||||
promise: string
|
||||
sessionID?: string
|
||||
}
|
||||
|
||||
const AGENT_LINE_PATTERN = /^Agent:[ \t]*(\S+)$/im
|
||||
const PROMISE_TAG_PATTERN = /<promise>[ \t]*(\S+?)[ \t]*<\/promise>/is
|
||||
const TASK_METADATA_PATTERN = /<task_metadata>[ \t]*([\s\S]*?)[ \t]*<\/task_metadata>/is
|
||||
const SESSION_ID_LINE_PATTERN = /^session_id:[ \t]*(\S+)$/im
|
||||
|
||||
export function parseOracleVerificationEvidence(text: string): OracleVerificationEvidence | undefined {
|
||||
const trimmedText = text.trim()
|
||||
if (!trimmedText) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const agentMatch = trimmedText.match(AGENT_LINE_PATTERN)
|
||||
if (!agentMatch) {
|
||||
return undefined
|
||||
}
|
||||
const agent = agentMatch[1]?.trim()
|
||||
if (!agent) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const promiseMatch = trimmedText.match(PROMISE_TAG_PATTERN)
|
||||
if (!promiseMatch) {
|
||||
return undefined
|
||||
}
|
||||
const promise = promiseMatch[1]?.trim()
|
||||
if (!promise) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const metadataMatch = trimmedText.match(TASK_METADATA_PATTERN)
|
||||
let sessionID: string | undefined
|
||||
if (metadataMatch) {
|
||||
const metadataContent = metadataMatch[1]
|
||||
const sessionIDMatch = metadataContent.match(SESSION_ID_LINE_PATTERN)
|
||||
if (sessionIDMatch) {
|
||||
sessionID = sessionIDMatch[1]?.trim()
|
||||
}
|
||||
}
|
||||
|
||||
return { agent, promise, sessionID }
|
||||
}
|
||||
|
||||
export function isOracleVerified(text: string): boolean {
|
||||
const evidence = parseOracleVerificationEvidence(text)
|
||||
if (!evidence) {
|
||||
return false
|
||||
}
|
||||
|
||||
const isOracleAgent = evidence.agent.toLowerCase() === "oracle"
|
||||
const isVerifiedPromise = evidence.promise === ULTRAWORK_VERIFICATION_PROMISE
|
||||
|
||||
return isOracleAgent && isVerifiedPromise
|
||||
}
|
||||
|
||||
export function extractOracleSessionID(text: string): string | undefined {
|
||||
const evidence = parseOracleVerificationEvidence(text)
|
||||
if (!evidence || evidence.agent.toLowerCase() !== "oracle") {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return evidence.sessionID
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { PluginInput } from "@opencode-ai/plugin"
|
||||
import { log } from "../../shared/logger"
|
||||
import { HOOK_NAME } from "./constants"
|
||||
import { ULTRAWORK_VERIFICATION_PROMISE } from "./constants"
|
||||
import { extractOracleSessionID, isOracleVerified } from "./oracle-verification-detector"
|
||||
import type { RalphLoopState } from "./types"
|
||||
import { handleFailedVerification } from "./verification-failure-handler"
|
||||
import { withTimeout } from "./with-timeout"
|
||||
@@ -11,13 +11,6 @@ type OpenCodeSessionMessage = {
|
||||
parts?: Array<{ type?: string; text?: string }>
|
||||
}
|
||||
|
||||
const ORACLE_AGENT_PATTERN = /Agent:\s*oracle/i
|
||||
const TASK_METADATA_SESSION_PATTERN = /<task_metadata>[\s\S]*?session_id:\s*([^\s<]+)[\s\S]*?<\/task_metadata>/i
|
||||
const VERIFIED_PROMISE_PATTERN = new RegExp(
|
||||
`<promise>\\s*${ULTRAWORK_VERIFICATION_PROMISE}\\s*<\\/promise>`,
|
||||
"i",
|
||||
)
|
||||
|
||||
function collectAssistantText(message: OpenCodeSessionMessage): string {
|
||||
if (!Array.isArray(message.parts)) {
|
||||
return ""
|
||||
@@ -67,12 +60,11 @@ async function detectOracleVerificationFromParentSession(
|
||||
}
|
||||
|
||||
const assistantText = collectAssistantText(message)
|
||||
if (!VERIFIED_PROMISE_PATTERN.test(assistantText) || !ORACLE_AGENT_PATTERN.test(assistantText)) {
|
||||
if (!isOracleVerified(assistantText)) {
|
||||
continue
|
||||
}
|
||||
|
||||
const sessionMatch = assistantText.match(TASK_METADATA_SESSION_PATTERN)
|
||||
const detectedOracleSessionID = sessionMatch?.[1]?.trim()
|
||||
const detectedOracleSessionID = extractOracleSessionID(assistantText)
|
||||
if (detectedOracleSessionID) {
|
||||
return detectedOracleSessionID
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user