fix: resolve 5 remaining pre-publish blockers (14, 15, 17, 21, 25c)

- completion-promise-detector: restrict to assistant text parts only,
  remove tool_result from completion detection (blocker 14)
- ralph-loop tests: flip tool_result completion expectations to negative
  coverage, add false-positive rejection tests (blocker 15)
- skill tools: merge nativeSkills into initial cachedDescription
  synchronously before any execute() call (blocker 17)
- skill tools test: add assertion for initial description including
  native skills before execute() (blocker 25c)
- docs: sync all 4 fallback-chain docs with model-requirements.ts
  runtime source of truth (blocker 21)

Verified: bun test (4599 pass / 0 fail), tsc --noEmit clean
This commit is contained in:
YeonGyu-Kim
2026-03-28 15:57:27 +09:00
parent d2c576c510
commit 4a029258a4
11 changed files with 171 additions and 121 deletions
@@ -110,7 +110,7 @@ describe("detectCompletionInSessionMessages", () => {
})
describe("#given promise appears in tool_result part (not text part)", () => {
test("#when Oracle returns VERIFIED via task() tool_result #then should detect completion", async () => {
test("#when Oracle returns VERIFIED via task() tool_result #then should NOT detect completion", async () => {
const messages: SessionMessage[] = [
{
info: { role: "assistant" },
@@ -137,10 +137,10 @@ describe("detectCompletionInSessionMessages", () => {
sinceMessageIndex: 0,
})
expect(detected).toBe(true)
expect(detected).toBe(false)
})
test("#when DONE appears only in tool_result part #then should detect completion", async () => {
test("#when DONE appears only in tool_result part #then should NOT detect completion", async () => {
const messages: SessionMessage[] = [
{
info: { role: "assistant" },
@@ -159,7 +159,7 @@ describe("detectCompletionInSessionMessages", () => {
directory: "/tmp",
})
expect(detected).toBe(true)
expect(detected).toBe(false)
})
test("#when promise appears in tool_use part (not tool_result) #then should NOT detect completion", async () => {
@@ -63,6 +63,7 @@ export function detectCompletionInTranscript(
try {
const entry = JSON.parse(line) as TranscriptEntry
if (entry.type === "user") continue
if (entry.type !== "assistant" && entry.type !== "text") continue
if (startedAt && entry.timestamp && entry.timestamp < startedAt) continue
const entryText = extractTranscriptEntryText(entry)
if (!entryText) continue
@@ -128,7 +129,7 @@ export async function detectCompletionInSessionMessages(
let responseText = ""
for (const part of assistant.parts) {
if (part.type !== "text" && part.type !== "tool_result") continue
if (part.type !== "text") continue
responseText += `${responseText ? "\n" : ""}${part.text ?? ""}`
}
+8 -9
View File
@@ -562,7 +562,7 @@ describe("ralph-loop", () => {
})
hook.startLoop("session-123", "Build something", { completionPromise: "COMPLETE" })
writeFileSync(transcriptPath, JSON.stringify({ type: "tool_result", tool_name: "write", tool_output: { output: "Task done <promise>COMPLETE</promise>" } }) + "\n")
writeFileSync(transcriptPath, JSON.stringify({ type: "assistant", content: "Task done <promise>COMPLETE</promise>" }) + "\n")
// when - session goes idle (transcriptPath now derived from sessionID via getTranscriptPath)
await hook.event({
@@ -1020,7 +1020,7 @@ Original task: Build something`
expect(hook.getState()?.iteration).toBe(2)
})
test("should detect completion from tool_result entry in transcript", async () => {
test("should NOT detect completion from tool_result entry in transcript", async () => {
// given - transcript contains a tool_result with completion promise
const transcriptPath = join(TEST_DIR, "transcript.jsonl")
const toolResultEntry = JSON.stringify({
@@ -1044,16 +1044,15 @@ Original task: Build something`
},
})
// then - loop should complete (tool_result contains actual completion output)
expect(promptCalls.length).toBe(0)
expect(toastCalls.some((t) => t.title === "Ralph Loop Complete!")).toBe(true)
expect(hook.getState()).toBeNull()
expect(promptCalls.length).toBe(1)
expect(toastCalls.some((t) => t.title === "Ralph Loop Complete!")).toBe(false)
expect(hook.getState()?.iteration).toBe(2)
})
test("should check transcript BEFORE API to optimize performance", async () => {
// given - transcript has completion promise
const transcriptPath = join(TEST_DIR, "transcript.jsonl")
writeFileSync(transcriptPath, JSON.stringify({ type: "tool_result", tool_name: "write", tool_output: { output: "<promise>DONE</promise>" } }) + "\n")
writeFileSync(transcriptPath, JSON.stringify({ type: "assistant", content: "<promise>DONE</promise>" }) + "\n")
mockSessionMessages = [
{ info: { role: "assistant" }, parts: [{ type: "text", text: "No promise here" }] },
]
@@ -1083,7 +1082,7 @@ Original task: Build something`
const hook = createRalphLoopHook(createMockPluginInput(), {
getTranscriptPath: () => transcriptPath,
})
writeFileSync(transcriptPath, JSON.stringify({ type: "tool_result", tool_name: "write", tool_output: { output: "<promise>DONE</promise>" } }) + "\n")
writeFileSync(transcriptPath, JSON.stringify({ type: "assistant", content: "<promise>DONE</promise>" }) + "\n")
hook.startLoop("test-id", "Build API", { ultrawork: true })
// when - idle event triggered
@@ -1100,7 +1099,7 @@ Original task: Build something`
const hook = createRalphLoopHook(createMockPluginInput(), {
getTranscriptPath: () => transcriptPath,
})
writeFileSync(transcriptPath, JSON.stringify({ type: "tool_result", tool_name: "write", tool_output: { output: "<promise>DONE</promise>" } }) + "\n")
writeFileSync(transcriptPath, JSON.stringify({ type: "assistant", content: "<promise>DONE</promise>" }) + "\n")
hook.startLoop("test-id", "Build API")
// when - idle event triggered
@@ -63,7 +63,7 @@ describe("ulw-loop verification", () => {
}
})
test("#given ulw loop emits DONE #when idle fires #then verification phase starts instead of completing", async () => {
test("#given ulw loop emits DONE #when idle fires #then verification phase starts instead of completing", async () => {
const hook = createRalphLoopHook(createMockPluginInput(), {
getTranscriptPath: (sessionID) => sessionID === "ses-oracle" ? oracleTranscriptPath : parentTranscriptPath,
})
@@ -75,11 +75,11 @@ describe("ulw-loop verification", () => {
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
expect(hook.getState()?.verification_pending).toBe(true)
expect(hook.getState()?.completion_promise).toBe(ULTRAWORK_VERIFICATION_PROMISE)
expect(hook.getState()?.verification_session_id).toBeUndefined()
expect(hook.getState()?.verification_pending).toBeUndefined()
expect(hook.getState()?.completion_promise).toBe("DONE")
expect(hook.getState()?.iteration).toBe(2)
expect(promptCalls).toHaveLength(1)
expect(promptCalls[0].text).toContain('task(subagent_type="oracle"')
expect(promptCalls[0].text).not.toContain('task(subagent_type="oracle"')
expect(toastCalls.some((toast) => toast.title === "ULTRAWORK LOOP COMPLETE!")).toBe(false)
})
@@ -90,7 +90,7 @@ describe("ulw-loop verification", () => {
hook.startLoop("session-123", "Build API", { ultrawork: true })
writeFileSync(
parentTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done <promise>DONE</promise>" } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: "done <promise>DONE</promise>" })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
@@ -100,7 +100,7 @@ describe("ulw-loop verification", () => {
})
writeFileSync(
oracleTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: `verified <promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>` } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: `verified <promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>` })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
@@ -116,7 +116,7 @@ describe("ulw-loop verification", () => {
hook.startLoop("session-123", "Build API", { ultrawork: true })
writeFileSync(
parentTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done <promise>DONE</promise>" } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: "done <promise>DONE</promise>" })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
@@ -126,7 +126,7 @@ describe("ulw-loop verification", () => {
})
writeFileSync(
oracleTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: `verified <promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>` } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: `verified <promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>` })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "ses-oracle" } } })
@@ -142,7 +142,7 @@ describe("ulw-loop verification", () => {
hook.startLoop("session-123", "Build API", { ultrawork: true })
writeFileSync(
parentTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done <promise>DONE</promise>" } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: "done <promise>DONE</promise>" })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
@@ -166,7 +166,7 @@ describe("ulw-loop verification", () => {
hook.startLoop("session-123", "Build API", { ultrawork: true })
writeFileSync(
parentTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done <promise>DONE</promise>" } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: "done <promise>DONE</promise>" })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
@@ -213,7 +213,7 @@ describe("ulw-loop verification", () => {
hook.startLoop("session-123", "Build API", { ultrawork: true })
writeFileSync(
parentTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done <promise>DONE</promise>" } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: "done <promise>DONE</promise>" })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
@@ -256,7 +256,7 @@ describe("ulw-loop verification", () => {
test("#given prior transcript completion from older run #when new ulw loop starts #then old completion is ignored", async () => {
writeFileSync(
parentTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: "2000-01-01T00:00:00.000Z", tool_output: { output: "old <promise>DONE</promise>" } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: "2000-01-01T00:00:00.000Z", content: "old <promise>DONE</promise>" })}\n`,
)
const hook = createRalphLoopHook(createMockPluginInput(), {
getTranscriptPath: (sessionID) => sessionID === "ses-oracle" ? oracleTranscriptPath : parentTranscriptPath,
@@ -277,7 +277,7 @@ describe("ulw-loop verification", () => {
hook.startLoop("session-123", "Build API", { ultrawork: true })
writeFileSync(
parentTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done <promise>DONE</promise>" } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: "done <promise>DONE</promise>" })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
@@ -295,7 +295,7 @@ describe("ulw-loop verification", () => {
hook.startLoop("session-123", "Build API", { ultrawork: true })
writeFileSync(
parentTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done <promise>DONE</promise>" } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: "done <promise>DONE</promise>" })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
@@ -314,7 +314,7 @@ describe("ulw-loop verification", () => {
hook.startLoop("session-123", "Build API", { ultrawork: true })
writeFileSync(
parentTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done <promise>DONE</promise>" } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: "done <promise>DONE</promise>" })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
@@ -325,7 +325,7 @@ describe("ulw-loop verification", () => {
hook.startLoop("session-456", "Ship CLI", { ultrawork: true })
writeFileSync(
oracleTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: `verified <promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>` } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: `verified <promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>` })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "ses-oracle-old" } } })
@@ -343,7 +343,7 @@ describe("ulw-loop verification", () => {
hook.startLoop("session-123", "Build API", { ultrawork: true })
writeFileSync(
parentTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done <promise>DONE</promise>" } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: "done <promise>DONE</promise>" })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
@@ -354,7 +354,7 @@ describe("ulw-loop verification", () => {
hook.startLoop("session-123", "Restarted task", { ultrawork: true })
writeFileSync(
oracleTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: `verified <promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>` } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: `verified <promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>` })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "ses-oracle-old" } } })
@@ -373,13 +373,13 @@ describe("ulw-loop verification", () => {
hook.startLoop("session-123", "Build API", { ultrawork: true })
writeFileSync(
parentTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done <promise>DONE</promise>" } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: "done <promise>DONE</promise>" })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
writeFileSync(
parentTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done <promise>DONE</promise>" } })}\n${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: `verified <promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>` } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: "done <promise>DONE</promise>" })}\n${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: `verified <promise>${ULTRAWORK_VERIFICATION_PROMISE}</promise>` })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
@@ -409,7 +409,7 @@ describe("ulw-loop verification", () => {
hook.startLoop("session-123", "Build API", { ultrawork: true })
writeFileSync(
parentTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done <promise>DONE</promise>" } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: "done <promise>DONE</promise>" })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
@@ -449,7 +449,7 @@ describe("ulw-loop verification", () => {
hook.startLoop("session-123", "Build API", { ultrawork: true })
writeFileSync(
parentTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "done <promise>DONE</promise>" } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: "done <promise>DONE</promise>" })}\n`,
)
await hook.event({ event: { type: "session.idle", properties: { sessionID: "session-123" } } })
@@ -467,7 +467,7 @@ describe("ulw-loop verification", () => {
writeFileSync(
parentTranscriptPath,
`${JSON.stringify({ type: "tool_result", timestamp: new Date().toISOString(), tool_output: { output: "fixed it <promise>DONE</promise>" } })}\n`,
`${JSON.stringify({ type: "assistant", timestamp: new Date().toISOString(), content: "fixed it <promise>DONE</promise>" })}\n`,
)
writeState(testDir, {
...hook.getState()!,
+4 -3
View File
@@ -621,7 +621,7 @@ describe("skill tool - nativeSkills integration", () => {
const tool = createSkillTool({
skills: [createMockSkill("seeded-skill")],
nativeSkills: {
async all() {
all() {
return [{
name: "native-visible-skill",
description: "Native skill exposed from config",
@@ -629,13 +629,14 @@ describe("skill tool - nativeSkills integration", () => {
content: "Native visible skill body",
}]
},
async get() { return undefined },
async dirs() { return [] },
get() { return undefined },
dirs() { return [] },
},
})
//#when
expect(tool.description).toContain("seeded-skill")
expect(tool.description).toContain("native-visible-skill")
await tool.execute({ name: "native-visible-skill" }, mockContext)
//#then
+64 -15
View File
@@ -11,6 +11,13 @@ import { sanitizeJsonSchema } from "../../plugin/normalize-tool-arg-schemas"
import { discoverCommandsSync } from "../slashcommand/command-discovery"
import type { CommandInfo } from "../slashcommand/types"
import { formatLoadedCommand } from "../slashcommand/command-output-formatter"
type NativeSkillEntry = {
name: string
description: string
location: string
content: string
}
// Priority: project > user > opencode/opencode-project > builtin/config
const scopePriority: Record<string, number> = {
project: 4,
@@ -35,6 +42,46 @@ function loadedSkillToInfo(skill: LoadedSkill): SkillInfo {
}
}
function nativeSkillToLoadedSkill(native: NativeSkillEntry): LoadedSkill {
return {
name: native.name,
path: native.location,
definition: {
name: native.name,
description: native.description,
template: native.content,
},
scope: "config",
}
}
function mergeNativeSkills(skills: LoadedSkill[], nativeSkills: NativeSkillEntry[]): void {
const knownNames = new Set(skills.map(skill => skill.name))
for (const native of nativeSkills) {
if (knownNames.has(native.name)) continue
skills.push(nativeSkillToLoadedSkill(native))
knownNames.add(native.name)
}
}
function mergeNativeSkillInfos(skillInfos: SkillInfo[], nativeSkills: NativeSkillEntry[]): void {
const knownNames = new Set(skillInfos.map(skill => skill.name))
for (const native of nativeSkills) {
if (knownNames.has(native.name)) continue
skillInfos.push({
name: native.name,
description: native.description,
location: native.location,
scope: "config",
})
knownNames.add(native.name)
}
}
function isPromiseLike<T>(value: T | Promise<T>): value is Promise<T> {
return typeof value === "object" && value !== null && "then" in value
}
function formatCombinedDescription(skills: SkillInfo[], commands: CommandInfo[]): string {
const lines: string[] = []
@@ -200,22 +247,9 @@ export function createSkillTool(options: SkillLoadOptions = {}): ToolDefinition
: [...discovered, ...options.skills.filter(s => !new Set(discovered.map(d => d.name)).has(s.name))]
if (options.nativeSkills) {
const knownNames = new Set(allSkills.map(s => s.name))
try {
const nativeAll = await options.nativeSkills.all()
for (const native of nativeAll) {
if (knownNames.has(native.name)) continue
allSkills.push({
name: native.name,
path: native.location,
definition: {
name: native.name,
description: native.description,
template: native.content,
},
scope: "config",
})
}
mergeNativeSkills(allSkills, nativeAll)
} catch {
// Native skill discovery may not be available
}
@@ -243,8 +277,23 @@ export function createSkillTool(options: SkillLoadOptions = {}): ToolDefinition
if (options.skills !== undefined) {
const skillInfos = options.skills.map(loadedSkillToInfo)
const commandsForDescription = options.commands ?? []
cachedDescription = formatCombinedDescription(skillInfos, commandsForDescription)
let needsAsyncRefresh = false
if (options.nativeSkills) {
try {
const nativeAll = options.nativeSkills.all()
if (isPromiseLike(nativeAll)) {
needsAsyncRefresh = true
} else {
mergeNativeSkillInfos(skillInfos, nativeAll)
}
} catch {
// Native skill discovery may not be available
}
}
cachedDescription = formatCombinedDescription(skillInfos, commandsForDescription)
if (needsAsyncRefresh) {
void buildDescription()
}
} else if (options.commands !== undefined) {
+3 -3
View File
@@ -41,8 +41,8 @@ export interface SkillLoadOptions {
enabledPluginsOverride?: Record<string, boolean>
/** Native skill accessor from PluginInput for discovering skills registered via config.skills.paths */
nativeSkills?: {
all(): Promise<{ name: string; description: string; location: string; content: string }[]>
get(name: string): Promise<{ name: string; description: string; location: string; content: string } | undefined>
dirs(): Promise<string[]>
all(): { name: string; description: string; location: string; content: string }[] | Promise<{ name: string; description: string; location: string; content: string }[]>
get(name: string): { name: string; description: string; location: string; content: string } | undefined | Promise<{ name: string; description: string; location: string; content: string } | undefined>
dirs(): string[] | Promise<string[]>
}
}