From 0c6805cc623df9e7b3eea63c152aaba154520092 Mon Sep 17 00:00:00 2001 From: YeonGyu-Kim Date: Mon, 11 May 2026 13:32:38 +0900 Subject: [PATCH] prompt(prometheus): add Oracle phase-gate verification between phases Inserts blocking Oracle verification todos (plan-1b / plan-2b / plan-6b in the canonical, plan-1b / plan-2b / plan-5b in the gpt and gemini variants) between each major Prometheus phase. Each gate is a single task(subagent_type=oracle) invocation that must return VERDICT: GO; NO-GO is a directive to fix the cited issues and rerun on the same Oracle session, not a license to skip. Adds a new 'Oracle Verification (Phase Gates)' section to plan-generation.ts with the concrete invocation prompts for each gate: phase 1 verifies interview completeness, phase 2 verifies the generated plan, phase 3 verifies plan readiness for execution before /start-work handoff. Also adds a plan-generation.test.ts smoke suite (9 cases) that pins the new todo ids, the section name, the GO/NO-GO format, the 'fix the cited issues' fallback, and the relative ordering. --- src/agents/prometheus/gemini.ts | 5 ++ src/agents/prometheus/gpt.ts | 5 ++ src/agents/prometheus/plan-generation.test.ts | 64 +++++++++++++++ src/agents/prometheus/plan-generation.ts | 82 +++++++++++++++++-- 4 files changed, 147 insertions(+), 9 deletions(-) create mode 100644 src/agents/prometheus/plan-generation.test.ts diff --git a/src/agents/prometheus/gemini.ts b/src/agents/prometheus/gemini.ts index ed617337b..73ac18881 100644 --- a/src/agents/prometheus/gemini.ts +++ b/src/agents/prometheus/gemini.ts @@ -205,14 +205,19 @@ CLEARANCE CHECKLIST (ALL must be YES to auto-transition): \`\`\`typescript TodoWrite([ { id: "plan-1", content: "Consult Metis for gap analysis", status: "pending", priority: "high" }, + { id: "plan-1b", content: "Oracle verification: phase 1 (interview completeness, scope, test strategy)", status: "pending", priority: "high" }, { id: "plan-2", content: "Generate plan to .sisyphus/plans/{name}.md", status: "pending", priority: "high" }, + { id: "plan-2b", content: "Oracle verification: phase 2 (plan compliance, parallelism, acceptance criteria)", status: "pending", priority: "high" }, { id: "plan-3", content: "Self-review: classify gaps", status: "pending", priority: "high" }, { id: "plan-4", content: "Present summary with decisions needed", status: "pending", priority: "high" }, { id: "plan-5", content: "Ask about high accuracy mode (Momus)", status: "pending", priority: "high" }, + { id: "plan-5b", content: "Oracle verification: phase 3 (plan readiness for execution)", status: "pending", priority: "high" }, { id: "plan-6", content: "Cleanup draft, guide to /start-work", status: "pending", priority: "medium" } ]) \`\`\` +Oracle verification gates (plan-1b, plan-2b, plan-5b) are blocking. Each is a single \`task(subagent_type="oracle", load_skills=[], run_in_background=false, prompt="...")\` invocation that must return \`VERDICT: GO\` before the workflow continues. \`NO-GO\` is a directive to fix the cited issues and rerun on the same Oracle session via \`task_id\`, not a license to skip. + ### Step 2: Consult Metis (MANDATORY) \`\`\`typescript diff --git a/src/agents/prometheus/gpt.ts b/src/agents/prometheus/gpt.ts index ec25b40a3..dcb4c45cd 100644 --- a/src/agents/prometheus/gpt.ts +++ b/src/agents/prometheus/gpt.ts @@ -192,14 +192,19 @@ CLEARANCE CHECKLIST (ALL must be YES to auto-transition): \`\`\`typescript TodoWrite([ { id: "plan-1", content: "Consult Metis for gap analysis", status: "pending", priority: "high" }, + { id: "plan-1b", content: "Oracle verification: phase 1 (interview completeness, scope, test strategy)", status: "pending", priority: "high" }, { id: "plan-2", content: "Generate plan to .sisyphus/plans/{name}.md", status: "pending", priority: "high" }, + { id: "plan-2b", content: "Oracle verification: phase 2 (plan compliance, parallelism, acceptance criteria)", status: "pending", priority: "high" }, { id: "plan-3", content: "Self-review: classify gaps (critical/minor/ambiguous)", status: "pending", priority: "high" }, { id: "plan-4", content: "Present summary with decisions needed", status: "pending", priority: "high" }, { id: "plan-5", content: "Ask about high accuracy mode (Momus review)", status: "pending", priority: "high" }, + { id: "plan-5b", content: "Oracle verification: phase 3 (plan readiness for execution)", status: "pending", priority: "high" }, { id: "plan-6", content: "Cleanup draft, guide to /start-work", status: "pending", priority: "medium" } ]) \`\`\` +Oracle verification gates (plan-1b, plan-2b, plan-5b) are blocking. Each is a single \`task(subagent_type="oracle", load_skills=[], run_in_background=false, prompt="...")\` invocation that must return \`VERDICT: GO\` before the workflow continues. \`NO-GO\` is a directive to fix the cited issues and rerun on the same Oracle session via \`task_id\`, not a license to skip. + ### Step 2: Consult Metis (MANDATORY) \`\`\`typescript diff --git a/src/agents/prometheus/plan-generation.test.ts b/src/agents/prometheus/plan-generation.test.ts new file mode 100644 index 000000000..cbc4f1838 --- /dev/null +++ b/src/agents/prometheus/plan-generation.test.ts @@ -0,0 +1,64 @@ +import { describe, it, expect } from "bun:test" +import { PROMETHEUS_PLAN_GENERATION } from "./plan-generation" + +describe("PROMETHEUS_PLAN_GENERATION oracle phase gates", () => { + describe("#given Prometheus plan generation prompt", () => { + describe("#when inspecting the registered todo list", () => { + it("#then includes plan-1b oracle verification after Metis", () => { + expect(PROMETHEUS_PLAN_GENERATION).toContain(`id: "plan-1b"`) + expect(PROMETHEUS_PLAN_GENERATION).toMatch(/plan-1b[^\n]*Oracle verification/i) + }) + + it("#then includes plan-2b oracle verification after plan generation", () => { + expect(PROMETHEUS_PLAN_GENERATION).toContain(`id: "plan-2b"`) + expect(PROMETHEUS_PLAN_GENERATION).toMatch(/plan-2b[^\n]*Oracle verification/i) + }) + + it("#then includes plan-6b oracle verification before handoff", () => { + expect(PROMETHEUS_PLAN_GENERATION).toContain(`id: "plan-6b"`) + expect(PROMETHEUS_PLAN_GENERATION).toMatch(/plan-6b[^\n]*Oracle verification/i) + }) + + it("#then preserves the existing plan-1 through plan-8 todos", () => { + for (const id of ["plan-1", "plan-2", "plan-3", "plan-4", "plan-5", "plan-6", "plan-7", "plan-8"]) { + expect(PROMETHEUS_PLAN_GENERATION, `${id} todo must remain`).toContain(`id: "${id}"`) + } + }) + }) + + describe("#when describing oracle invocations", () => { + it("#then provides concrete task() calls for all three phase gates", () => { + const oracleInvocations = PROMETHEUS_PLAN_GENERATION.match(/subagent_type="oracle"/g) ?? [] + expect(oracleInvocations.length).toBeGreaterThanOrEqual(3) + }) + + it("#then names a dedicated Oracle Verification section", () => { + expect(PROMETHEUS_PLAN_GENERATION).toContain("Oracle Verification (Phase Gates)") + }) + + it("#then declares each gate is blocking with GO/NO-GO verdict format", () => { + expect(PROMETHEUS_PLAN_GENERATION).toContain("VERDICT: GO/NO-GO") + expect(PROMETHEUS_PLAN_GENERATION.toLowerCase()).toContain("blocking") + }) + + it("#then forbids skipping the gate on NO-GO", () => { + const lower = PROMETHEUS_PLAN_GENERATION.toLowerCase() + expect(lower).toMatch(/no-go is not an excuse to skip|fix the cited issues/) + }) + }) + + describe("#when describing the updated workflow", () => { + it("#then orders the gates after their respective phases", () => { + const idxPlan1b = PROMETHEUS_PLAN_GENERATION.indexOf(`id: "plan-1b"`) + const idxPlan2 = PROMETHEUS_PLAN_GENERATION.indexOf(`id: "plan-2"`) + const idxPlan2b = PROMETHEUS_PLAN_GENERATION.indexOf(`id: "plan-2b"`) + const idxPlan6 = PROMETHEUS_PLAN_GENERATION.indexOf(`id: "plan-6"`) + const idxPlan6b = PROMETHEUS_PLAN_GENERATION.indexOf(`id: "plan-6b"`) + + expect(idxPlan1b, "plan-1b must precede plan-2 (gate runs before next phase)").toBeLessThan(idxPlan2) + expect(idxPlan2b, "plan-2b must follow plan-2").toBeGreaterThan(idxPlan2) + expect(idxPlan6b, "plan-6b must follow plan-6").toBeGreaterThan(idxPlan6) + }) + }) + }) +}) diff --git a/src/agents/prometheus/plan-generation.ts b/src/agents/prometheus/plan-generation.ts index e44d5428f..5e974c881 100644 --- a/src/agents/prometheus/plan-generation.ts +++ b/src/agents/prometheus/plan-generation.ts @@ -27,11 +27,14 @@ export const PROMETHEUS_PLAN_GENERATION = `# PHASE 2: PLAN GENERATION (Auto-Tran // IMMEDIATELY upon trigger detection - NO EXCEPTIONS todoWrite([ { id: "plan-1", content: "Consult Metis for gap analysis (auto-proceed)", status: "pending", priority: "high" }, + { id: "plan-1b", content: "Oracle verification: phase 1 (interview completeness, requirements clarity, scope boundaries)", status: "pending", priority: "high" }, { id: "plan-2", content: "Generate work plan to .sisyphus/plans/{name}.md", status: "pending", priority: "high" }, + { id: "plan-2b", content: "Oracle verification: phase 2 (plan compliance with constraints, parallelism, acceptance criteria)", status: "pending", priority: "high" }, { id: "plan-3", content: "Self-review: classify gaps (critical/minor/ambiguous)", status: "pending", priority: "high" }, { id: "plan-4", content: "Present summary with auto-resolved items and decisions needed", status: "pending", priority: "high" }, { id: "plan-5", content: "If decisions needed: wait for user, update plan", status: "pending", priority: "high" }, { id: "plan-6", content: "Ask user about high accuracy mode (Momus review)", status: "pending", priority: "high" }, + { id: "plan-6b", content: "Oracle verification: phase 3 (plan readiness for execution before high-accuracy or handoff)", status: "pending", priority: "high" }, { id: "plan-7", content: "If high accuracy: Submit to Momus and iterate until OKAY", status: "pending", priority: "medium" }, { id: "plan-8", content: "Delete draft file and guide user to /start-work {name}", status: "pending", priority: "medium" } ]) @@ -39,20 +42,81 @@ todoWrite([ **WHY THIS IS CRITICAL:** - User sees exactly what steps remain -- Prevents skipping crucial steps like Metis consultation +- Prevents skipping crucial steps like Metis consultation and Oracle phase gates - Creates accountability for each phase - Enables recovery if session is interrupted **WORKFLOW:** -1. Trigger detected → **IMMEDIATELY** TodoWrite (plan-1 through plan-8) +1. Trigger detected → **IMMEDIATELY** TodoWrite (plan-1 through plan-8, including plan-1b / plan-2b / plan-6b) 2. Mark plan-1 as \`in_progress\` → Consult Metis (auto-proceed, no questions) -3. Mark plan-2 as \`in_progress\` → Generate plan immediately -4. Mark plan-3 as \`in_progress\` → Self-review and classify gaps -5. Mark plan-4 as \`in_progress\` → Present summary (with auto-resolved/defaults/decisions) -6. Mark plan-5 as \`in_progress\` → If decisions needed, wait for user and update plan -7. Mark plan-6 as \`in_progress\` → Ask high accuracy question -8. Continue marking todos as you progress -9. NEVER skip a todo. NEVER proceed without updating status. +3. Mark plan-1b as \`in_progress\` → Run Oracle phase-1 verification (see "Oracle Verification (Phase Gates)" below). Must produce VERDICT: GO before continuing. +4. Mark plan-2 as \`in_progress\` → Generate plan immediately +5. Mark plan-2b as \`in_progress\` → Run Oracle phase-2 verification on the saved plan file. Must produce VERDICT: GO before continuing. +6. Mark plan-3 as \`in_progress\` → Self-review and classify gaps +7. Mark plan-4 as \`in_progress\` → Present summary (with auto-resolved/defaults/decisions) +8. Mark plan-5 as \`in_progress\` → If decisions needed, wait for user and update plan +9. Mark plan-6 as \`in_progress\` → Ask high accuracy question +10. Mark plan-6b as \`in_progress\` → Run Oracle phase-3 verification on the final plan (with any user-driven edits applied). Must produce VERDICT: GO before handoff. +11. Continue marking todos as you progress +12. NEVER skip a todo. NEVER proceed without updating status. **Oracle phase gates are blocking: if Oracle returns NO-GO, fix the cited issues and rerun the same Oracle verification on the same session.** + +## Oracle Verification (Phase Gates) + +Three blocking phase gates use the Oracle agent (read-only consultant). Each gate is a single \`task(subagent_type="oracle", load_skills=[], run_in_background=false, prompt="...")\` invocation. The Oracle must return VERDICT: GO before the workflow continues. NO-GO is not an excuse to skip — fix the cited issues and rerun on the same session via \`task_id\`. + +### plan-1b: phase 1 verification (after Metis, before plan generation) + +\`\`\`typescript +task( + subagent_type="oracle", + load_skills=[], + run_in_background=false, + prompt=\`Verify Prometheus phase 1 (interview) is complete and consistent. Read the draft at .sisyphus/drafts/{name}.md and Metis's findings recorded in this session. Confirm: + 1. Core objective is unambiguous (one sentence, no hidden alternates). + 2. Scope IN / Scope OUT are both explicit. + 3. Test strategy is decided (TDD / tests-after / none + agent QA). + 4. No outstanding user questions remain. + 5. No requirement contradicts the codebase patterns surfaced by explore/librarian. + Return: \\\`CHECK [N/5] PASS | VERDICT: GO/NO-GO\\\` plus, on NO-GO, a numbered list of issues that block.\` +) +\`\`\` + +### plan-2b: phase 2 verification (after plan generation, before self-review) + +\`\`\`typescript +task( + subagent_type="oracle", + load_skills=[], + run_in_background=false, + prompt=\`Verify Prometheus phase 2 (plan generation). Read .sisyphus/plans/{name}.md end to end. Confirm: + 1. Every TODO item carries acceptance criteria with concrete success conditions. + 2. Each task has a recommended agent profile and a Wave assignment. + 3. Parallelism is maximized (waves contain 3-8 tasks except where dependencies force fewer). + 4. Must Have / Must NOT Have lists exist and are consistent with the interview record. + 5. No task requires assumptions about business logic without cited evidence. + 6. Plan path is .sisyphus/plans/, not docs/ or plans/. + Return: \\\`CHECK [N/6] PASS | VERDICT: GO/NO-GO\\\` plus, on NO-GO, file:line citations for each blocking issue.\` +) +\`\`\` + +### plan-6b: phase 3 verification (after high-accuracy decision, before handoff) + +\`\`\`typescript +task( + subagent_type="oracle", + load_skills=[], + run_in_background=false, + prompt=\`Verify the plan at .sisyphus/plans/{name}.md is ready for execution by /start-work. Confirm: + 1. Any decisions surfaced in the user summary have been resolved and reflected in the plan. + 2. The final-wave reviewer set (F1-F4) is present and addressable. + 3. Commit strategy and verification commands are stated. + 4. The plan is internally consistent after the most recent edits. + 5. If high-accuracy mode was selected, Momus's last verdict is OKAY (or the loop is still in progress). + Return: \\\`CHECK [N/5] PASS | VERDICT: GO/NO-GO\\\` plus, on NO-GO, what to fix.\` +) +\`\`\` + +**Why phase gates are mandatory:** Metis catches what Prometheus might have missed during interview. Oracle catches what Prometheus might be wrong about. Both run before code is touched. NO-GO is a directive to fix, not a license to abandon the gate. ## Pre-Generation: Metis Consultation (MANDATORY)