diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/checkpoint.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/checkpoint.test.ts new file mode 100644 index 000000000..e58d9b592 --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/checkpoint.test.ts @@ -0,0 +1,213 @@ +// biome-ignore-all format: keep the single mandated checkpoint spec under the pure LOC budget. +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { checkpointUlwLoop } from "../src/checkpoint.js"; +import { ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE } from "../src/goal-status.js"; +import { ulwLoopBriefPath, ulwLoopDir, ulwLoopLedgerPath } from "../src/paths.js"; +import { writePlan } from "../src/plan-io.js"; +import type { UlwLoopItem, UlwLoopLedgerEntry, UlwLoopPlan, UlwLoopSuccessCriterion } from "../src/types.js"; +import { UlwLoopError } from "../src/types.js"; + +const NOW = "2026-05-23T00:00:00.000Z"; +const QUALITY_GATE_PATH = join(process.cwd(), "test", "fixtures", "sample-quality-gate.json"); + +function criterion(id: string, status: UlwLoopSuccessCriterion["status"]): UlwLoopSuccessCriterion { + return { id, scenario: `${id} scenario`, userModel: "happy", expectedEvidence: `${id} proof`, capturedEvidence: status === "pass" ? `${id} passed` : null, status }; +} + +function goal(overrides: Partial = {}): UlwLoopItem { + return { id: "G001", title: "Build auth", objective: "Implement JWT auth endpoint", status: "in_progress", successCriteria: [criterion("C001", "pass"), criterion("C002", "pass"), criterion("C003", "pass")], attempt: 1, createdAt: NOW, updatedAt: NOW, ...overrides }; +} + +function plan(goals: UlwLoopItem[], overrides: Partial = {}): UlwLoopPlan { + const result: UlwLoopPlan = { version: 1, createdAt: NOW, updatedAt: NOW, briefPath: ".omo/ulw-loop/brief.md", goalsPath: ".omo/ulw-loop/goals.json", ledgerPath: ".omo/ulw-loop/ledger.jsonl", codexGoalMode: "aggregate", codexObjective: ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE, goals }; + Object.assign(result, overrides); + const activeGoalId = goals.find((candidate) => candidate.status === "in_progress")?.id; + if (result.activeGoalId === undefined && activeGoalId !== undefined) result.activeGoalId = activeGoalId; + return result; +} + +async function samplePlan(overrides: Partial = {}): Promise { + const fixture: UlwLoopPlan = JSON.parse(await readFile(new URL("./fixtures/sample-plan.json", import.meta.url), "utf8")); + return plan(fixture.goals.map((item, index) => goal({ ...item, attempt: index + 1, createdAt: NOW, updatedAt: NOW })), overrides); +} + +async function repoWith(seed: UlwLoopPlan): Promise { + const repo = await mkdtemp(join(tmpdir(), "ug-checkpoint-")); + await mkdir(ulwLoopDir(repo), { recursive: true }); + await writePlan(repo, seed); + return repo; +} + +function snapshot(status: "active" | "complete", objective = ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE): string { + return JSON.stringify({ goal: { objective, status } }); +} + +async function lastLedger(repo: string): Promise { + const last = (await readFile(ulwLoopLedgerPath(repo), "utf8")).trim().split(/\r?\n/).at(-1); + if (last === undefined) throw new Error("expected ledger entry"); + const entry: UlwLoopLedgerEntry = JSON.parse(last); + return entry; +} + +async function expectCode(action: () => Promise, code: string): Promise { + try { + await action(); + } catch (error) { + expect(error).toBeInstanceOf(UlwLoopError); + if (!(error instanceof UlwLoopError)) throw error; + expect(error.code).toBe(code); + return; + } + throw new Error("Expected UlwLoopError"); +} + +function passGoal(id: string, overrides: Partial = {}): UlwLoopItem { + return goal({ id, successCriteria: [criterion("C001", "pass"), criterion("C002", "pass"), criterion("C003", "pass")], ...overrides }); +} + +describe("checkpointUlwLoop status=complete criteria gate", () => { + it("THROWS ulw_loop_criteria_not_all_pass when any criterion is pending", async () => { + const repo = await repoWith(await samplePlan({ goals: [goal({ successCriteria: [criterion("C001", "pass"), criterion("C002", "pending"), criterion("C003", "pass")] })] })); + await expectCode(() => checkpointUlwLoop(repo, { goalId: "G001", status: "complete", evidence: "done" }), "ulw_loop_criteria_not_all_pass"); + }); + + it("THROWS when any criterion is fail or blocked", async () => { + for (const status of ["fail", "blocked"] satisfies UlwLoopSuccessCriterion["status"][]) { + const repo = await repoWith(plan([goal({ successCriteria: [criterion("C001", "pass"), criterion("C002", status), criterion("C003", "pass")] })])); + await expectCode(() => checkpointUlwLoop(repo, { goalId: "G001", status: "complete", evidence: "done" }), "ulw_loop_criteria_not_all_pass"); + } + }); + + it("THROWS when criteria list is empty", async () => { + const repo = await repoWith(plan([goal({ successCriteria: [] }), goal({ id: "G002", status: "pending" })])); + await expectCode(() => checkpointUlwLoop(repo, { goalId: "G001", status: "complete", evidence: "done", codexGoalJson: snapshot("active") }), "ulw_loop_criteria_not_all_pass"); + }); + + it("ACCEPTS complete when ALL criteria pass (with valid snapshot)", async () => { + const repo = await repoWith(plan([passGoal("G001"), goal({ id: "G002", status: "pending" })])); + const result = await checkpointUlwLoop(repo, { goalId: "G001", status: "complete", evidence: "implementation done and tests passed", codexGoalJson: snapshot("active") }); + expect(result.goal.status).toBe("complete"); + expect((await lastLedger(repo)).kind).toBe("goal_completed"); + }); +}); + +describe("checkpointUlwLoop reconciliation (status=complete)", () => { + it("succeeds when snapshot objective matches expected (aggregate active)", async () => { + const repo = await repoWith(plan([passGoal("G001"), goal({ id: "G002", status: "pending" })])); + await expect(checkpointUlwLoop(repo, { goalId: "G001", status: "complete", evidence: "work complete and validation passed", codexGoalJson: snapshot("active") })).resolves.toMatchObject({ goal: { status: "complete" } }); + }); + + it("throws on mismatched objective", async () => { + const repo = await repoWith(plan([passGoal("G001"), goal({ id: "G002", status: "pending" })])); + await expectCode(() => checkpointUlwLoop(repo, { goalId: "G001", status: "complete", evidence: "work complete and validation passed", codexGoalJson: snapshot("active", "wrong objective") }), "ulw_loop_codex_snapshot_mismatch"); + }); + + it("throws on mismatched status (snapshot complete when expected active)", async () => { + const repo = await repoWith(plan([passGoal("G001"), goal({ id: "G002", status: "pending" })])); + await expectCode(() => checkpointUlwLoop(repo, { goalId: "G001", status: "complete", evidence: "work complete and validation passed", codexGoalJson: snapshot("complete") }), "ulw_loop_codex_snapshot_mismatch"); + }); +}); + +describe("checkpointUlwLoop final story", () => { + it("requires quality-gate-json for the final goal complete", async () => { + const repo = await repoWith(plan([passGoal("G001", { status: "complete" }), passGoal("G002")], { activeGoalId: "G002" })); + await expectCode(() => checkpointUlwLoop(repo, { goalId: "G002", status: "complete", evidence: "final work complete and validation passed", codexGoalJson: snapshot("complete") }), "ULW_LOOP_QUALITY_GATE_INVALID"); + }); + + it("accepts final story when quality gate JSON includes valid criteriaCoverage", async () => { + const repo = await repoWith(plan([passGoal("G001", { status: "complete" }), passGoal("G002")], { activeGoalId: "G002" })); + const result = await checkpointUlwLoop(repo, { goalId: "G002", status: "complete", evidence: "final work complete and validation passed", codexGoalJson: snapshot("complete"), qualityGateJson: QUALITY_GATE_PATH }); + expect(result.aggregateCompletion?.status).toBe("complete"); + expect(result.plan.aggregateCompletion?.status).toBe("complete"); + }); + + it("ACCEPTS complete when task-scoped completed Codex objective maps to the ulw-loop brief", async () => { + const taskObjective = "Fix ulw-loop objective mismatch and install local ulw"; + const repo = await repoWith(plan([passGoal("G001")], { activeGoalId: "G001" })); + await writeFile(ulwLoopBriefPath(repo), `${taskObjective}\n`, "utf8"); + + const result = await checkpointUlwLoop(repo, { + goalId: "G001", + status: "complete", + evidence: "final implementation complete and quality gate passed", + codexGoalJson: snapshot("complete", taskObjective), + qualityGateJson: QUALITY_GATE_PATH, + }); + + expect(result.aggregateCompletion?.status).toBe("complete"); + expect(result.ledgerEntry.kind).toBe("aggregate_completed"); + }); + + it("explains final task-scoped objective mapping when completed Codex objective is unrelated", async () => { + const repo = await repoWith(plan([passGoal("G001")], { activeGoalId: "G001" })); + await writeFile(ulwLoopBriefPath(repo), "Fix ulw-loop objective mismatch and install local ulw\n", "utf8"); + + await expect( + checkpointUlwLoop(repo, { + goalId: "G001", + status: "complete", + evidence: "final implementation complete and quality gate passed", + codexGoalJson: snapshot("complete", "unrelated completed task"), + qualityGateJson: QUALITY_GATE_PATH, + }), + ).rejects.toThrow("Final task-scoped aggregate reconciliation"); + }); +}); + +describe("checkpointUlwLoop status=failed", () => { + it("sets goal.status=failed, goal.failedAt, appends ledger", async () => { + const repo = await repoWith(plan([goal({ successCriteria: [criterion("C001", "pending")] })])); + const result = await checkpointUlwLoop(repo, { goalId: "G001", status: "failed", evidence: "tests failed" }); + expect(result.goal.status).toBe("failed"); + expect(result.goal.failedAt).toMatch(/^\d{4}-\d{2}-\d{2}T/u); + expect((await lastLedger(repo)).kind).toBe("goal_failed"); + }); + + it("classifies external authorization blocker signatures", async () => { + const repo = await repoWith(plan([goal()])); + const result = await checkpointUlwLoop(repo, { goalId: "G001", status: "failed", evidence: "ghcr.io returned 401 authentication required because token missing" }); + expect(result.goal.blockerSignature).toBe("GHCR_PULL_ACCESS:HTTP_401_ANONYMOUS:GHCR_VISIBILITY_OR_CREDENTIAL_REQUIRED"); + }); + + it("after 3 same-signature blockers, marks needs_user_decision + nonRetriable", async () => { + const repo = await repoWith(plan([goal({ id: "G001", status: "failed", blockerSignature: "EXTERNAL_AUTHORIZATION_REQUIRED" }), goal({ id: "G002", status: "blocked", blockerSignature: "EXTERNAL_AUTHORIZATION_REQUIRED" }), goal({ id: "G003" })], { activeGoalId: "G003" })); + const result = await checkpointUlwLoop(repo, { goalId: "G003", status: "failed", evidence: "Registry returned 401 because credentials are missing" }); + expect(result.goal.status).toBe("needs_user_decision"); + expect(result.goal.nonRetriable).toBe(true); + }); + + it("skips the criteria gate for failed status", async () => { + const repo = await repoWith(plan([goal({ successCriteria: [criterion("C001", "pending")] })])); + await expect(checkpointUlwLoop(repo, { goalId: "G001", status: "failed", evidence: "not done" })).resolves.toMatchObject({ goal: { status: "failed" } }); + }); +}); + +describe("checkpointUlwLoop status=blocked", () => { + it("preserves blocker fields + appends ledger", async () => { + const repo = await repoWith(plan([goal()])); + const result = await checkpointUlwLoop(repo, { goalId: "G001", status: "blocked", evidence: "ghcr.io requires token and credentials are missing" }); + expect(result.goal.status).toBe("blocked"); + expect(result.goal.blockedReason).toContain("ghcr.io"); + expect(result.goal.blockerSignature).toContain("GHCR_PULL_ACCESS"); + expect((await lastLedger(repo)).kind).toBe("goal_blocked"); + }); + + it("skips the criteria gate for blocked status", async () => { + const repo = await repoWith(plan([goal({ successCriteria: [criterion("C001", "pending")] })])); + await expect(checkpointUlwLoop(repo, { goalId: "G001", status: "blocked", evidence: "waiting for approval" })).resolves.toMatchObject({ goal: { status: "blocked" } }); + }); +}); + +describe("checkpointUlwLoop rebrand", () => { + it("does not emit legacy brand token in any returned text or ledger payload", async () => { + const repo = await repoWith(plan([passGoal("G001"), goal({ id: "G002", status: "pending" })])); + const result = await checkpointUlwLoop(repo, { goalId: "G001", status: "complete", evidence: "implementation done in .omo/ulw-loop/goals.json for G001 and validation passed", codexGoalJson: snapshot("active") }); + const forbidden = ["o", "m", "x"].join(""); + const payload = `${JSON.stringify(result)}\n${await readFile(ulwLoopLedgerPath(repo), "utf8")}`.toLowerCase(); + expect(payload).not.toContain(forbidden); + }); +}); diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/cli-commands.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/cli-commands.test.ts new file mode 100644 index 000000000..6905d5676 --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/cli-commands.test.ts @@ -0,0 +1,375 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { ulwLoopCommand } from "../src/cli-commands.ts"; +import { ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE } from "../src/goal-status.js"; + +let testDir: string; +let out: string[]; +let err: string[]; +let originalCodexSessionId: string | undefined; +let originalCodexThreadId: string | undefined; +let originalOmoSessionId: string | undefined; + +beforeEach(async () => { + testDir = await mkdtemp(join(tmpdir(), "ug-cli-")); + out = []; + err = []; + originalCodexSessionId = process.env["CODEX_SESSION_ID"]; + originalCodexThreadId = process.env["CODEX_THREAD_ID"]; + originalOmoSessionId = process.env["OMO_ULW_LOOP_SESSION_ID"]; + delete process.env["CODEX_SESSION_ID"]; + delete process.env["CODEX_THREAD_ID"]; + delete process.env["OMO_ULW_LOOP_SESSION_ID"]; + vi.spyOn(process, "cwd").mockReturnValue(testDir); + vi.spyOn(process.stdout, "write").mockImplementation((chunk: string | Uint8Array): boolean => { + out.push(chunk.toString()); + return true; + }); + vi.spyOn(process.stderr, "write").mockImplementation((chunk: string | Uint8Array): boolean => { + err.push(chunk.toString()); + return true; + }); +}); + +afterEach(async () => { + vi.restoreAllMocks(); + if (originalCodexSessionId === undefined) delete process.env["CODEX_SESSION_ID"]; + else process.env["CODEX_SESSION_ID"] = originalCodexSessionId; + if (originalCodexThreadId === undefined) delete process.env["CODEX_THREAD_ID"]; + else process.env["CODEX_THREAD_ID"] = originalCodexThreadId; + if (originalOmoSessionId === undefined) delete process.env["OMO_ULW_LOOP_SESSION_ID"]; + else process.env["OMO_ULW_LOOP_SESSION_ID"] = originalOmoSessionId; + await rm(testDir, { recursive: true, force: true }); +}); + +function resetOutput(): void { + out = []; + err = []; +} +function stdoutJson(): Record { + return JSON.parse(out.join("")); +} +function codexSnapshot(status: "active" | "complete" = "active"): string { + return JSON.stringify({ goal: { objective: ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE, status } }); +} + +async function createPlan(brief = "- Goal A\n- Goal B"): Promise> { + resetOutput(); + expect(await ulwLoopCommand(["create-goals", "--brief", brief, "--json"])).toBe(0); + const parsed = stdoutJson(); + resetOutput(); + return parsed; +} + +async function passCriterion(goalId: string, criterionId: string): Promise { + expect( + await ulwLoopCommand([ + "record-evidence", + "--goal-id", + goalId, + "--criterion-id", + criterionId, + "--status", + "pass", + "--evidence", + `${criterionId} observable proof`, + ]), + ).toBe(0); + resetOutput(); +} + +describe("ulwLoopCommand help", () => { + it("prints usage when no subcommand", async () => { + expect(await ulwLoopCommand([])).toBe(0); + expect(out.join("")).toContain("omo ulw-loop"); + }); +}); + +describe("ulwLoopCommand create-goals", () => { + it("creates plan + writes 3 artifacts + seeds criteria per goal", async () => { + const code = await ulwLoopCommand(["create-goals", "--brief", "- Goal A\n- Goal B", "--json"]); + + expect(code).toBe(0); + const parsed = stdoutJson(); + expect(parsed).toMatchObject({ ok: true }); + expect(parsed).toHaveProperty("plan.goals.0.successCriteria.0.id", "C001"); + expect(await readFile(join(testDir, ".omo/ulw-loop/brief.md"), "utf8")).toContain("Goal A"); + expect(await readFile(join(testDir, ".omo/ulw-loop/goals.json"), "utf8")).toContain("successCriteria"); + expect(await readFile(join(testDir, ".omo/ulw-loop/ledger.jsonl"), "utf8")).toContain("plan_created"); + }); + + it("#given two session ids #when creating goals #then writes isolated session-scoped plans", async () => { + expect(await ulwLoopCommand(["create-goals", "--session-id", "session-A", "--brief", "- Alpha", "--json"])).toBe( + 0, + ); + resetOutput(); + + expect(await ulwLoopCommand(["create-goals", "--session-id", "session-B", "--brief", "- Beta", "--json"])).toBe( + 0, + ); + resetOutput(); + + expect(await readFile(join(testDir, ".omo/ulw-loop/session-A/goals.json"), "utf8")).toContain("Alpha"); + expect(await readFile(join(testDir, ".omo/ulw-loop/session-B/goals.json"), "utf8")).toContain("Beta"); + + expect(await ulwLoopCommand(["status", "--session-id", "session-A", "--json"])).toBe(0); + expect(stdoutJson()).toMatchObject({ + plan: { goalsPath: ".omo/ulw-loop/session-A/goals.json", goals: [{ title: "Alpha" }] }, + }); + expect(out.join("")).not.toContain("Beta"); + }); + + it("#given Codex thread env #when creating goals #then uses the thread as the session scope", async () => { + process.env["CODEX_THREAD_ID"] = "thread-123"; + + expect(await ulwLoopCommand(["create-goals", "--brief", "- Thread scoped", "--json"])).toBe(0); + resetOutput(); + + expect(await readFile(join(testDir, ".omo/ulw-loop/thread-123/goals.json"), "utf8")).toContain("Thread scoped"); + expect(await ulwLoopCommand(["status", "--json"])).toBe(0); + expect(stdoutJson()).toHaveProperty("plan.goalsPath", ".omo/ulw-loop/thread-123/goals.json"); + }); + + it("#given Codex thread env and explicit session id #when creating goals #then the explicit session wins", async () => { + process.env["CODEX_THREAD_ID"] = "thread-123"; + + expect( + await ulwLoopCommand(["create-goals", "--session-id", "manual-456", "--brief", "- Manual scoped", "--json"]), + ).toBe(0); + + expect(await readFile(join(testDir, ".omo/ulw-loop/manual-456/goals.json"), "utf8")).toContain("Manual scoped"); + }); +}); + +describe("ulwLoopCommand status", () => { + it("prints plan summary including criteria counts", async () => { + await createPlan(); + + expect(await ulwLoopCommand(["status"])).toBe(0); + expect(out.join("")).toContain("criteria: 0/6 pass"); + }); +}); + +describe("ulwLoopCommand complete-goals", () => { + it("starts the next goal and returns a Codex instruction", async () => { + await createPlan(); + + expect(await ulwLoopCommand(["complete-goals", "--json"])).toBe(0); + expect(stdoutJson()).toMatchObject({ + ok: true, + goal: { status: "in_progress" }, + instruction: { json: { status: "active" } }, + }); + }); +}); + +describe("ulwLoopCommand record-evidence", () => { + it("records evidence + returns updated criterion", async () => { + await createPlan(); + + expect( + await ulwLoopCommand([ + "record-evidence", + "--goal-id", + "G001-goal-a", + "--criterion-id", + "C001", + "--status", + "pass", + "--evidence", + "curl passed", + "--json", + ]), + ).toBe(0); + expect(stdoutJson()).toMatchObject({ + ok: true, + criterion: { id: "C001", status: "pass", capturedEvidence: "curl passed" }, + }); + }); + + it("returns 1 + error on unknown goal-id", async () => { + await createPlan(); + + expect( + await ulwLoopCommand([ + "record-evidence", + "--goal-id", + "G404", + "--criterion-id", + "C001", + "--status", + "pass", + "--evidence", + "x", + ]), + ).toBe(1); + expect(err.join("")).toContain("[ulw-loop]"); + }); + + it("returns 1 + error on missing flags", async () => { + expect( + await ulwLoopCommand(["record-evidence", "--criterion-id", "C001", "--status", "pass", "--evidence", "x"]), + ).toBe(1); + expect(err.join("")).toContain("Missing --goal-id"); + }); +}); + +describe("ulwLoopCommand criteria", () => { + it("lists criteria for a goal", async () => { + await createPlan(); + + expect(await ulwLoopCommand(["criteria", "--goal-id", "G001-goal-a"])).toBe(0); + expect(out.join("")).toContain("C001"); + expect(out.join("")).toContain("happy"); + }); + + it("supports --json output", async () => { + await createPlan(); + + expect(await ulwLoopCommand(["criteria", "--goal-id", "G001-goal-a", "--json"])).toBe(0); + expect(stdoutJson()).toMatchObject({ ok: true, goalId: "G001-goal-a" }); + expect(stdoutJson()).toHaveProperty("criteria.0.id", "C001"); + }); +}); + +describe("ulwLoopCommand checkpoint", () => { + it("REJECTS status=complete when criteria pending", async () => { + await createPlan(); + + expect( + await ulwLoopCommand([ + "checkpoint", + "--goal-id", + "G001-goal-a", + "--status", + "complete", + "--evidence", + "x", + "--codex-goal-json", + codexSnapshot(), + ]), + ).toBe(1); + expect(err.join("").toLowerCase()).toContain("criteria"); + }); + + it("ACCEPTS when all criteria pass", async () => { + await createPlan(); + await passCriterion("G001-goal-a", "C001"); + await passCriterion("G001-goal-a", "C002"); + await passCriterion("G001-goal-a", "C003"); + + expect( + await ulwLoopCommand([ + "checkpoint", + "--goal-id", + "G001-goal-a", + "--status", + "complete", + "--evidence", + "implementation done and validation passed", + "--codex-goal-json", + codexSnapshot(), + "--json", + ]), + ).toBe(0); + expect(stdoutJson()).toHaveProperty("goal.status", "complete"); + }); + + it("#given failed checkpoint without codex goal json #when recorded through CLI #then marks the goal failed", async () => { + await createPlan(); + + expect( + await ulwLoopCommand([ + "checkpoint", + "--goal-id", + "G001-goal-a", + "--status", + "failed", + "--evidence", + "implementation failed and validation captured", + "--json", + ]), + ).toBe(0); + + expect(stdoutJson()).toMatchObject({ ok: true, goal: { id: "G001-goal-a", status: "failed" } }); + }); + + it("#given blocked checkpoint without codex goal json #when recorded through CLI #then marks the goal blocked", async () => { + await createPlan(); + + expect( + await ulwLoopCommand([ + "checkpoint", + "--goal-id", + "G002-goal-b", + "--status", + "blocked", + "--evidence", + "waiting for external approval", + "--json", + ]), + ).toBe(0); + + expect(stdoutJson()).toMatchObject({ ok: true, goal: { id: "G002-goal-b", status: "blocked" } }); + }); +}); + +describe("ulwLoopCommand steer", () => { + it("dispatches to the steering engine", async () => { + await createPlan(); + + expect( + await ulwLoopCommand([ + "steer", + "--kind", + "add_subgoal", + "--title", + "Extra", + "--objective", + "Do extra", + "--evidence", + "user requested it", + "--rationale", + "keeps plan accurate", + "--json", + ]), + ).toBe(0); + expect(stdoutJson()).toMatchObject({ + ok: true, + accepted: true, + plan: { + goals: [ + { id: "G001-goal-a" }, + { id: "G002-goal-b" }, + { id: "G003", title: "Extra", successCriteria: [{ id: "C001" }, { id: "C002" }, { id: "C003" }] }, + ], + }, + }); + }); +}); + +describe("ulwLoopCommand add-goal", () => { + it("appends a pending goal", async () => { + await createPlan(); + + expect(await ulwLoopCommand(["add-goal", "--title", "Later", "--objective", "Do later", "--json"])).toBe(0); + expect(stdoutJson()).toMatchObject({ ok: true, goal: { title: "Later", status: "pending" } }); + }); +}); + +describe("ulwLoopCommand unknown", () => { + it("returns 1 + prints help on unknown subcommand", async () => { + expect(await ulwLoopCommand(["wat"])).toBe(1); + expect(out.join("")).toContain("omo ulw-loop"); + }); +}); + +describe("ulwLoopCommand error handling", () => { + it("returns 1 + prints [ulw-loop] prefix on UlwLoopError", async () => { + expect(await ulwLoopCommand(["status"])).toBe(1); + expect(err.join("")).toContain("[ulw-loop]"); + }); +}); diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/cli-helpers.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/cli-helpers.test.ts new file mode 100644 index 000000000..0f515f53a --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/cli-helpers.test.ts @@ -0,0 +1,250 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +import { + hasFlag, + parseGoalArg, + parseRecordEvidenceArgs, + positionalText, + readJsonInput, + readRepeated, + readValue, +} from "../src/cli-arg-parser.js"; +import { normalizeCodexGoalMode, printStatus, ULW_LOOP_HELP } from "../src/cli-output.js"; +import type { UlwLoopItem, UlwLoopPlan, UlwLoopSuccessCriterion } from "../src/types.js"; +import { UlwLoopError } from "../src/types.js"; + +const NOW = "2026-05-23T00:00:00.000Z"; + +function criterion(overrides: Partial = {}): UlwLoopSuccessCriterion { + return { + id: "C001", + scenario: "happy path returns 200", + userModel: "happy", + expectedEvidence: "HTTP 200", + capturedEvidence: null, + status: "pending", + ...overrides, + }; +} + +function goal(overrides: Partial = {}): UlwLoopItem { + return { + id: "G001", + title: "Auth endpoint", + objective: "Build JWT auth", + status: "in_progress", + successCriteria: [ + criterion({ id: "C001", status: "pass" }), + criterion({ id: "C002", status: "pass" }), + criterion({ id: "C003" }), + ], + attempt: 1, + createdAt: NOW, + updatedAt: NOW, + ...overrides, + }; +} + +function plan(overrides: Partial = {}): UlwLoopPlan { + return { + version: 1, + createdAt: NOW, + updatedAt: NOW, + briefPath: ".omo/ulw-loop/brief.md", + goalsPath: ".omo/ulw-loop/goals.json", + ledgerPath: ".omo/ulw-loop/ledger.jsonl", + activeGoalId: "G001", + goals: [goal()], + ...overrides, + }; +} + +function captureStdout(action: () => void): string { + let output = ""; + const write = vi.spyOn(process.stdout, "write").mockImplementation((chunk: string | Uint8Array): boolean => { + output += chunk.toString(); + return true; + }); + action(); + write.mockRestore(); + return output; +} + +describe("hasFlag", () => { + it("returns true for present flag", () => { + expect(hasFlag(["status", "--json"], "--json")).toBe(true); + }); + + it("returns false otherwise", () => { + expect(hasFlag(["status"], "--json")).toBe(false); + }); +}); + +describe("readValue", () => { + it("returns value after flag", () => { + expect(readValue(["criteria", "--goal-id", "G001"], "--goal-id")).toBe("G001"); + }); + + it("returns undefined when absent", () => { + expect(readValue(["criteria"], "--goal-id")).toBeUndefined(); + }); + + it("returns undefined when flag has no following value", () => { + expect(readValue(["criteria", "--goal-id"], "--goal-id")).toBeUndefined(); + }); +}); + +describe("readRepeated", () => { + it("collects all occurrences", () => { + expect(readRepeated(["create-goals", "--goal", "A", "--goal=B"], "--goal")).toEqual(["A", "B"]); + }); +}); + +describe("parseGoalArg", () => { + it("returns value of --goal-id or --goal", () => { + expect(parseGoalArg(["criteria", "--goal", "G002"])).toBe("G002"); + expect(parseGoalArg(["criteria", "--goal-id", "G001"])).toBe("G001"); + }); +}); + +describe("positionalText", () => { + it("returns joined positional args after subcommand", () => { + expect(positionalText(["create-goals", "Build", "auth", "--json", "--brief", "ignored"])).toBe("Build auth"); + }); +}); + +describe("readJsonInput", () => { + it("parses inline JSON when value looks like JSON", async () => { + await expect(readJsonInput('{"ok":true}')).resolves.toEqual({ ok: true }); + }); + + it("reads from file path", async () => { + const dir = await mkdtemp(join(tmpdir(), "ug-cli-json-")); + try { + const file = join(dir, "input.json"); + await writeFile(file, JSON.stringify({ fromFile: true }), "utf8"); + + await expect(readJsonInput(file)).resolves.toEqual({ fromFile: true }); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it("returns undefined when value is undefined", async () => { + await expect(readJsonInput(undefined)).resolves.toBeUndefined(); + }); +}); + +describe("parseRecordEvidenceArgs", () => { + it("parses --goal-id + --criterion-id + --status + --evidence", () => { + expect( + parseRecordEvidenceArgs([ + "record-evidence", + "--goal-id", + "G001", + "--criterion-id", + "C001", + "--status", + "pass", + "--evidence", + "curl 200", + ]), + ).toEqual({ goalId: "G001", criterionId: "C001", status: "pass", evidence: "curl 200" }); + }); + + it("throws when goal-id missing", () => { + expect(() => + parseRecordEvidenceArgs(["record-evidence", "--criterion-id", "C001", "--status", "pass", "--evidence", "x"]), + ).toThrow(UlwLoopError); + }); + + it("throws when status is not pass|fail|blocked", () => { + expect(() => + parseRecordEvidenceArgs([ + "record-evidence", + "--goal-id", + "G001", + "--criterion-id", + "C001", + "--status", + "skip", + "--evidence", + "x", + ]), + ).toThrow(UlwLoopError); + }); + + it("includes optional --notes when present", () => { + expect( + parseRecordEvidenceArgs([ + "record-evidence", + "--goal-id", + "G001", + "--criterion-id", + "C001", + "--status", + "blocked", + "--evidence", + "auth missing", + "--notes", + "waiting", + ]), + ).toMatchObject({ notes: "waiting" }); + }); +}); + +describe("ULW_LOOP_HELP", () => { + it("mentions omo ulw-loop + every subcommand", () => { + expect(ULW_LOOP_HELP).toContain("omo ulw-loop"); + expect(ULW_LOOP_HELP).toContain("create-goals"); + expect(ULW_LOOP_HELP).toContain("complete-goals"); + expect(ULW_LOOP_HELP).toContain("status"); + expect(ULW_LOOP_HELP).toContain("checkpoint"); + expect(ULW_LOOP_HELP).toContain("steer"); + expect(ULW_LOOP_HELP).toContain("record-evidence"); + expect(ULW_LOOP_HELP).toContain("criteria"); + expect(ULW_LOOP_HELP).toContain("add-goal"); + expect(ULW_LOOP_HELP).toContain("record-review-blockers"); + }); + + it("never mentions the legacy typo", () => { + const typo = ["o", "m", "x"].join(""); + + expect(ULW_LOOP_HELP).not.toMatch(new RegExp(typo, "i")); + }); +}); + +describe("printStatus", () => { + it("shows criteria P/T per goal", () => { + const output = captureStdout(() => printStatus(plan())); + + expect(output).toContain("criteria: 2/3"); + }); + + it("shows aggregate counts", () => { + const output = captureStdout(() => + printStatus(plan({ goals: [goal(), goal({ id: "G002", successCriteria: [criterion({ status: "pass" })] })] })), + ); + + expect(output).toContain("total goals: 2"); + expect(output).toContain("criteria: 3/4 pass"); + }); +}); + +describe("normalizeCodexGoalMode", () => { + it("returns aggregate when undefined", () => { + expect(normalizeCodexGoalMode(undefined)).toBe("aggregate"); + }); + + it("returns the explicit value when valid", () => { + expect(normalizeCodexGoalMode("per_story")).toBe("per_story"); + }); + + it("throws UlwLoopError when invalid", () => { + expect(() => normalizeCodexGoalMode("per-story")).toThrow(UlwLoopError); + }); +}); diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/cli-steering.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/cli-steering.test.ts new file mode 100644 index 000000000..16566f061 --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/cli-steering.test.ts @@ -0,0 +1,407 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +import { + normalizeSteeringProposal, + parseSteeringKind, + parseSteeringProposal, + parseSteeringSource, + printSteerResult, +} from "../src/cli-steering.js"; +import type { SteerUlwLoopResult, UlwLoopPlan } from "../src/types.js"; +import { UlwLoopError } from "../src/types.js"; + +const NOW = "2026-05-23T00:00:00.000Z"; + +function plan(): UlwLoopPlan { + return { + version: 1, + createdAt: NOW, + updatedAt: NOW, + briefPath: ".omo/ulw-loop/brief.md", + goalsPath: ".omo/ulw-loop/goals.json", + ledgerPath: ".omo/ulw-loop/ledger.jsonl", + goals: [], + }; +} + +function steerResult(overrides: Partial = {}): SteerUlwLoopResult { + return { + plan: plan(), + accepted: true, + audit: { + kind: "add_subgoal", + source: "cli", + targetGoalIds: ["G001"], + evidence: "x", + rationale: "y", + invariant: { + accepted: true, + structuralInvariantAccepted: true, + evidenceBackedNecessity: true, + noEasierCompletion: true, + rejectedReasons: [], + }, + }, + rejectedReasons: [], + deduped: false, + ...overrides, + }; +} + +function captureStdout(action: () => void): string { + let output = ""; + const write = vi.spyOn(process.stdout, "write").mockImplementation((chunk: string | Uint8Array): boolean => { + output += chunk.toString(); + return true; + }); + action(); + write.mockRestore(); + return output; +} + +describe("parseSteeringKind", () => { + it("returns valid kind from --kind", () => { + expect(parseSteeringKind(["--kind", "add_subgoal"])).toBe("add_subgoal"); + }); + + it("accepts revise_criterion", () => { + expect(parseSteeringKind(["--kind", "revise_criterion"])).toBe("revise_criterion"); + }); + + it("throws when --kind missing", () => { + expect(() => parseSteeringKind([])).toThrow(UlwLoopError); + }); + + it("throws when kind unknown", () => { + expect(() => parseSteeringKind(["--kind", "bogus"])).toThrow(UlwLoopError); + }); +}); + +describe("parseSteeringSource", () => { + it("defaults to cli", () => { + expect(parseSteeringSource([])).toBe("cli"); + }); + + it("returns explicit value", () => { + expect(parseSteeringSource(["--source", "user_prompt_submit"])).toBe("user_prompt_submit"); + }); +}); + +describe("parseSteeringProposal add_subgoal", () => { + it("builds proposal from required flags", async () => { + const p = await parseSteeringProposal([ + "--kind", + "add_subgoal", + "--title", + " New ", + "--objective", + " Build ", + "--evidence", + " x ", + "--rationale", + " y ", + ]); + + expect(p).toMatchObject({ + kind: "add_subgoal", + source: "cli", + title: "New", + objective: "Build", + evidence: "x", + rationale: "y", + }); + }); + + it("throws when --title missing", async () => { + await expect( + parseSteeringProposal([ + "--kind", + "add_subgoal", + "--objective", + "Build", + "--evidence", + "x", + "--rationale", + "y", + ]), + ).rejects.toThrow(UlwLoopError); + }); + + it("throws when --evidence missing", async () => { + await expect( + parseSteeringProposal(["--kind", "add_subgoal", "--title", "New", "--objective", "Build", "--rationale", "y"]), + ).rejects.toThrow(UlwLoopError); + }); +}); + +describe("parseSteeringProposal revise_criterion", () => { + it("builds proposal with goal, criterion, scenario, evidence, and rationale", async () => { + const p = await parseSteeringProposal([ + "--kind", + "revise_criterion", + "--goal-id", + "G001", + "--criterion-id", + "C002", + "--scenario", + "new scenario", + "--evidence", + "x", + "--rationale", + "y", + ]); + + expect(p.kind).toBe("revise_criterion"); + expect(p.goalId).toBe("G001"); + expect(p.targetGoalId).toBe("G001"); + expect(p.criterionId).toBe("C002"); + expect(p.scenario).toBe("new scenario"); + }); + + it("accepts --expected-evidence as an update field", async () => { + const p = await parseSteeringProposal([ + "--kind", + "revise_criterion", + "--goal-id", + "G001", + "--criterion-id", + "C002", + "--expected-evidence", + "new evidence", + "--evidence", + "x", + "--rationale", + "y", + ]); + + expect(p.expectedEvidence).toBe("new evidence"); + }); + + it("accepts --user-model as an update field", async () => { + const p = await parseSteeringProposal([ + "--kind", + "revise_criterion", + "--goal-id", + "G001", + "--criterion-id", + "C002", + "--user-model", + "edge", + "--evidence", + "x", + "--rationale", + "y", + ]); + + expect(p.userModel).toBe("edge"); + }); + + it("throws when none of scenario/expected-evidence/user-model provided", async () => { + await expect( + parseSteeringProposal([ + "--kind", + "revise_criterion", + "--goal-id", + "G001", + "--criterion-id", + "C002", + "--evidence", + "x", + "--rationale", + "y", + ]), + ).rejects.toThrow(UlwLoopError); + }); + + it("throws when goal-id missing", async () => { + await expect( + parseSteeringProposal([ + "--kind", + "revise_criterion", + "--criterion-id", + "C002", + "--scenario", + "s", + "--evidence", + "x", + "--rationale", + "y", + ]), + ).rejects.toThrow(UlwLoopError); + }); + + it("throws when criterion-id missing", async () => { + await expect( + parseSteeringProposal([ + "--kind", + "revise_criterion", + "--goal-id", + "G001", + "--scenario", + "s", + "--evidence", + "x", + "--rationale", + "y", + ]), + ).rejects.toThrow(UlwLoopError); + }); +}); + +describe("parseSteeringProposal split_subgoal", () => { + it("reads --children from inline JSON", async () => { + const p = await parseSteeringProposal([ + "--kind", + "split_subgoal", + "--goal-id", + "G001", + "--children", + '[{"title":"A","objective":"Do A"}]', + "--evidence", + "x", + "--rationale", + "y", + ]); + + expect(p.childGoals).toEqual([{ title: "A", objective: "Do A" }]); + }); + + it("reads --children from JSON file path", async () => { + const dir = await mkdtemp(join(tmpdir(), "ug-steer-")); + try { + const file = join(dir, "children.json"); + await writeFile(file, '[{"title":"B","objective":"Do B"}]', "utf8"); + + const p = await parseSteeringProposal([ + "--kind", + "split_subgoal", + "--goal-id", + "G001", + "--children", + file, + "--evidence", + "x", + "--rationale", + "y", + ]); + + expect(p.childGoals).toEqual([{ title: "B", objective: "Do B" }]); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); + +describe("parseSteeringProposal reorder_pending", () => { + it("reads --order from inline JSON array", async () => { + const p = await parseSteeringProposal([ + "--kind", + "reorder_pending", + "--order", + '["G002","G001"]', + "--evidence", + "x", + "--rationale", + "y", + ]); + + expect(p.pendingOrder).toEqual(["G002", "G001"]); + }); +}); + +describe("parseSteeringProposal remaining kinds", () => { + it("builds revise_pending_wording proposal", async () => { + const p = await parseSteeringProposal([ + "--kind", + "revise_pending_wording", + "--goal-id", + "G001", + "--title", + "New", + "--evidence", + "x", + "--rationale", + "y", + ]); + + expect(p).toMatchObject({ kind: "revise_pending_wording", targetGoalId: "G001", revisedTitle: "New" }); + }); + + it("builds mark_blocked_superseded proposal with replacements", async () => { + const p = await parseSteeringProposal([ + "--kind", + "mark_blocked_superseded", + "--goal-id", + "G001", + "--replacements", + '[{"title":"C","objective":"Do C"}]', + "--evidence", + "x", + "--rationale", + "y", + ]); + + expect(p).toMatchObject({ + kind: "mark_blocked_superseded", + targetGoalId: "G001", + childGoals: [{ title: "C", objective: "Do C" }], + }); + }); +}); + +describe("parseSteeringProposal annotate_ledger", () => { + it("builds minimal proposal", async () => { + const p = await parseSteeringProposal(["--kind", "annotate_ledger", "--evidence", "x", "--rationale", "y"]); + + expect(p).toMatchObject({ kind: "annotate_ledger", source: "cli", evidence: "x", rationale: "y" }); + }); +}); + +describe("normalizeSteeringProposal", () => { + it("trims string fields", () => { + const p = normalizeSteeringProposal({ + kind: "revise_criterion", + source: "cli", + goalId: " G001 ", + targetGoalId: " G001 ", + criterionId: " C002 ", + evidence: " x ", + rationale: " y ", + scenario: " z ", + }); + + expect(p).toMatchObject({ + goalId: "G001", + targetGoalId: "G001", + criterionId: "C002", + evidence: "x", + rationale: "y", + scenario: "z", + }); + }); + + it("rejects empty evidence after trim", () => { + expect(() => + normalizeSteeringProposal({ kind: "annotate_ledger", source: "cli", evidence: " ", rationale: "y" }), + ).toThrow(UlwLoopError); + }); +}); + +describe("printSteerResult", () => { + it("prints JSON when json=true", () => { + const output = captureStdout(() => printSteerResult(steerResult(), true)); + + expect(JSON.parse(output)).toMatchObject({ accepted: true, deduped: false, audit: { kind: "add_subgoal" } }); + }); + + it("prints human-readable when json=false", () => { + const output = captureStdout(() => printSteerResult(steerResult(), false)); + + expect(output).toContain("ulw-loop steer: accepted add_subgoal"); + expect(output).toContain("ulw-loop status"); + }); +}); diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/codex-goal-instruction.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/codex-goal-instruction.test.ts new file mode 100644 index 000000000..46a84e2b2 --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/codex-goal-instruction.test.ts @@ -0,0 +1,169 @@ +import { describe, expect, it } from "vitest"; + +import { buildCodexGoalInstruction } from "../src/codex-goal-instruction.js"; +import { ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE } from "../src/goal-status.js"; +import type { UlwLoopItem, UlwLoopPlan, UlwLoopSuccessCriterion } from "../src/types.js"; + +const NOW = "2026-05-23T00:00:00.000Z"; + +function makeCriterion(overrides: Partial = {}): UlwLoopSuccessCriterion { + return { + id: "C001", + scenario: "happy path", + userModel: "happy", + expectedEvidence: "observable proof", + capturedEvidence: null, + status: "pending", + ...overrides, + }; +} + +function makeGoal(overrides: Partial = {}): UlwLoopItem { + return { + id: "G001", + title: "Goal one", + objective: "Complete goal one", + status: "pending", + successCriteria: [], + attempt: 1, + createdAt: NOW, + updatedAt: NOW, + ...overrides, + }; +} + +function makePlan(overrides: Partial = {}): UlwLoopPlan { + return { + version: 1, + createdAt: NOW, + updatedAt: NOW, + briefPath: ".omo/ulw-loop/brief.md", + goalsPath: ".omo/ulw-loop/goals.json", + ledgerPath: ".omo/ulw-loop/ledger.jsonl", + goals: [], + ...overrides, + }; +} + +describe("buildCodexGoalInstruction aggregate mode", () => { + it("references the aggregate handoff and the .omo/ulw-loop/goals.json artifact", () => { + const { text } = buildCodexGoalInstruction({ plan: makePlan({ codexGoalMode: "aggregate" }), goal: makeGoal() }); + expect(text).toContain("aggregate"); + expect(text).toContain(".omo/ulw-loop/goals.json"); + }); + + it("given aggregate mode when rendering create_goal payload then omits numeric limits", () => { + const { json, text } = buildCodexGoalInstruction({ + plan: makePlan({ codexGoalMode: "aggregate" }), + goal: makeGoal(), + }); + expect(json).toEqual({ + objective: ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE, + status: "active", + }); + expect(text).toContain("objective and status only"); + expect(text).toContain("Goals are unlimited"); + expect(text).not.toMatch(/token[_-]?budget/i); + }); + + it("instructs not to call update_goal mid-aggregate when not final", () => { + const { text } = buildCodexGoalInstruction({ + plan: makePlan({ codexGoalMode: "aggregate" }), + goal: makeGoal(), + isFinal: false, + }); + expect(text).toMatch(/do not.*update_goal/i); + }); + + it("includes quality gate instruction when isFinal", () => { + const { text } = buildCodexGoalInstruction({ + plan: makePlan({ codexGoalMode: "aggregate" }), + goal: makeGoal(), + isFinal: true, + }); + expect(text).toMatch(/quality gate/i); + }); + + it("#given a scoped plan #when rendering final commands #then includes the session id option", () => { + const { text } = buildCodexGoalInstruction({ + plan: makePlan({ + codexGoalMode: "aggregate", + goalsPath: ".omo/ulw-loop/session-A/goals.json", + ledgerPath: ".omo/ulw-loop/session-A/ledger.jsonl", + }), + goal: makeGoal(), + isFinal: true, + }); + + expect(text).toContain("record-review-blockers --session-id session-A"); + expect(text).toContain("checkpoint --session-id session-A"); + expect(text).toContain("complete-goals --session-id session-A --retry-failed"); + }); +}); + +describe("buildCodexGoalInstruction per_story mode", () => { + it("uses the goal's own objective for create_goal", () => { + const goal = makeGoal({ objective: "Build the auth service" }); + const { text } = buildCodexGoalInstruction({ plan: makePlan({ codexGoalMode: "per_story" }), goal }); + expect(text).toContain("Build the auth service"); + }); +}); + +describe("buildCodexGoalInstruction criteria section", () => { + it("lists every successCriteria entry with id + scenario + status", () => { + const goal = makeGoal({ + successCriteria: [ + makeCriterion({ + id: "C001", + scenario: "happy login", + userModel: "happy", + expectedEvidence: "200 OK", + status: "pending", + }), + makeCriterion({ + id: "C002", + scenario: "invalid creds", + userModel: "edge", + expectedEvidence: "401", + status: "pass", + }), + makeCriterion({ + id: "C003", + scenario: "no regression /health", + userModel: "regression", + expectedEvidence: "/health unaffected", + status: "fail", + }), + ], + }); + + const { text } = buildCodexGoalInstruction({ plan: makePlan(), goal }); + + expect(text).toContain("C001"); + expect(text).toContain("happy login"); + expect(text).toContain("pending"); + expect(text).toContain("C002"); + expect(text).toContain("pass"); + expect(text).toContain("C003"); + expect(text).toContain("fail"); + }); + + it("highlights pending criteria as remaining work", () => { + const goal = makeGoal({ successCriteria: [makeCriterion({ id: "C001", status: "pending" })] }); + const { text } = buildCodexGoalInstruction({ plan: makePlan(), goal }); + expect(text).toMatch(/remaining|pending/i); + }); +}); + +describe("buildCodexGoalInstruction rebrand audit", () => { + it("emits no legacy brand references in any rendered string", () => { + const legacyBrand = ["o", "m", "x"].join(""); + const { text } = buildCodexGoalInstruction({ plan: makePlan(), goal: makeGoal() }); + expect(text).not.toMatch(new RegExp(legacyBrand, "i")); + }); + + it("references .omo/ulw-loop in artifact paths", () => { + const { text } = buildCodexGoalInstruction({ plan: makePlan({ codexGoalMode: "aggregate" }), goal: makeGoal() }); + expect(text).toContain(".omo/ulw-loop"); + }); +}); diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/codex-goal-snapshot.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/codex-goal-snapshot.test.ts new file mode 100644 index 000000000..b70f1ffaf --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/codex-goal-snapshot.test.ts @@ -0,0 +1,156 @@ +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { beforeEach, describe, expect, it } from "vitest"; +import { + CodexGoalSnapshotError, + formatCodexGoalReconciliation, + parseCodexGoalSnapshot, + readCodexGoalSnapshotInput, + reconcileCodexGoalSnapshot, +} from "../src/codex-goal-snapshot.ts"; + +describe("parseCodexGoalSnapshot", () => { + it("returns available snapshot from { goal: { ... } } JSON", () => { + // given + const payload = { goal: { objective: "X", status: "active" } }; + + // when + const snapshot = parseCodexGoalSnapshot(payload); + + // then + expect(snapshot.available).toBe(true); + expect(snapshot.objective).toBe("X"); + expect(snapshot.status).toBe("active"); + }); + + it("ignores remaining token budget fields from goal snapshots", () => { + // given + const payload = { goal: { objective: "X", status: "active" }, remainingTokens: 123 }; + + // when + const snapshot = parseCodexGoalSnapshot(payload); + + // then + expect("remainingTokens" in snapshot).toBe(false); + }); + + it("returns unavailable snapshot from null", () => { + // when + const snapshot = parseCodexGoalSnapshot(null); + + // then + expect(snapshot.available).toBe(false); + }); + + it("returns unavailable snapshot from malformed payload", () => { + // when + const snapshot = parseCodexGoalSnapshot({ wrong: "shape" }); + + // then + expect(snapshot.available).toBe(false); + expect(snapshot.status).toBe("unknown"); + }); +}); + +describe("readCodexGoalSnapshotInput", () => { + let dir = ""; + + beforeEach(async () => { + // given + dir = await mkdtemp(join(tmpdir(), "ug-snap-")); + }); + + it("parses inline JSON string", async () => { + // when + const snapshot = await readCodexGoalSnapshotInput('{"goal":{"objective":"X","status":"active"}}'); + + // then + expect(snapshot?.available).toBe(true); + expect(snapshot?.objective).toBe("X"); + }); + + it("reads from file path", async () => { + // given + const filePath = join(dir, "snap.json"); + await writeFile(filePath, '{"goal":{"objective":"X","status":"complete"}}', "utf8"); + + // when + const snapshot = await readCodexGoalSnapshotInput(filePath); + + // then + expect(snapshot?.available).toBe(true); + expect(snapshot?.status).toBe("complete"); + }); + + it("reads from sample fixture path", async () => { + // given + const filePath = join(process.cwd(), "test", "fixtures", "codex-goal-snapshot.json"); + + // when + const snapshot = await readCodexGoalSnapshotInput(filePath); + + // then + expect(snapshot?.available).toBe(true); + expect(snapshot?.objective).toBe("Complete the durable ulw-loop plan"); + }); + + it("throws CodexGoalSnapshotError when input is neither JSON nor a path", async () => { + // when/then + await expect(readCodexGoalSnapshotInput("not json and not a path")).rejects.toThrow(CodexGoalSnapshotError); + }); +}); + +describe("reconcileCodexGoalSnapshot", () => { + it("returns ok=true when snapshot matches expected", () => { + // when + const reconciliation = reconcileCodexGoalSnapshot( + { available: true, objective: "X", status: "active", raw: null }, + { expectedObjective: "X" }, + ); + + // then + expect(reconciliation.ok).toBe(true); + expect(reconciliation.errors).toHaveLength(0); + }); + + it("reports error when objective mismatches", () => { + // when + const reconciliation = reconcileCodexGoalSnapshot( + { available: true, objective: "X", status: "active", raw: null }, + { expectedObjective: "Y" }, + ); + + // then + expect(reconciliation.ok).toBe(false); + expect(reconciliation.errors.length).toBeGreaterThan(0); + }); + + it("reports error when status mismatches", () => { + // when + const reconciliation = reconcileCodexGoalSnapshot( + { available: true, objective: "X", status: "active", raw: null }, + { expectedObjective: "X", allowedStatuses: ["complete"] }, + ); + + // then + expect(reconciliation.ok).toBe(false); + expect(reconciliation.errors.length).toBeGreaterThan(0); + }); +}); + +describe("formatCodexGoalReconciliation", () => { + it("renders errors joined", () => { + // given + const reconciliation = reconcileCodexGoalSnapshot( + { available: true, objective: "X", status: "active", raw: null }, + { expectedObjective: "Y", allowedStatuses: ["complete"] }, + ); + + // when + const formatted = formatCodexGoalReconciliation(reconciliation); + + // then + expect(formatted).toMatch(/objective|status/i); + }); +}); diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/codex-hook.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/codex-hook.test.ts new file mode 100644 index 000000000..e250f11b3 --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/codex-hook.test.ts @@ -0,0 +1,275 @@ +import { mkdir, mkdtemp, readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable, Writable } from "node:stream"; +import { describe, expect, it } from "vitest"; + +import { + applyPreToolUseGoalBudgetGuard, + applyUserPromptUlwLoopSteering, + type PreToolUsePayload, + parseUserPromptSubmitPayload, + runPreToolUseGoalBudgetGuardCli, + runUlwLoopHookCli, + type UserPromptSubmitPayload, +} from "../src/codex-hook.js"; +import { ulwLoopDir, ulwLoopLedgerPath } from "../src/paths.js"; +import { writePlan } from "../src/plan-io.js"; +import type { UlwLoopPlan } from "../src/types.js"; + +const NOW = "2026-05-23T00:00:00.000Z"; +const DEFAULT_SESSION_ID = "s1"; + +async function bootstrapPlanRepo(): Promise { + const repoRoot = await mkdtemp(join(tmpdir(), "ug-hook-")); + await mkdir(ulwLoopDir(repoRoot, { sessionId: DEFAULT_SESSION_ID }), { recursive: true }); + await writePlan(repoRoot, samplePlan(), { sessionId: DEFAULT_SESSION_ID }); + return repoRoot; +} + +function samplePlan(): UlwLoopPlan { + return { + version: 1, + createdAt: NOW, + updatedAt: NOW, + briefPath: ".omo/ulw-loop/brief.md", + goalsPath: ".omo/ulw-loop/goals.json", + ledgerPath: ".omo/ulw-loop/ledger.jsonl", + goals: [ + { + id: "G001", + title: "Build hook", + objective: "Apply safe steering directives from Codex hooks.", + status: "pending", + successCriteria: [], + attempt: 0, + createdAt: NOW, + updatedAt: NOW, + }, + ], + }; +} + +function payload(prompt: string, cwd: string): UserPromptSubmitPayload { + return { cwd, hook_event_name: "UserPromptSubmit", prompt, session_id: DEFAULT_SESSION_ID }; +} + +function preToolPayload(toolName: string, toolInput: unknown): PreToolUsePayload { + return { + cwd: "/repo", + hook_event_name: "PreToolUse", + model: "gpt-5.5", + permission_mode: "default", + session_id: "s1", + tool_input: toolInput, + tool_name: toolName, + tool_use_id: "call-1", + transcript_path: null, + turn_id: "turn-1", + }; +} + +function payloadWithRuntimeEvent(hookEventName: string): UserPromptSubmitPayload { + const input = payload( + 'OMO_ULW_LOOP_STEER: {"kind":"annotate_ledger","source":"user_prompt_submit","evidence":"x","rationale":"y"}', + "/tmp", + ); + Object.defineProperty(input, "hook_event_name", { value: hookEventName }); + return input; +} + +function captureStdout(): { readonly stdout: Writable; readonly read: () => string } { + let captured = ""; + const stdout = new Writable({ + write(chunk: unknown, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void { + captured += chunk instanceof Buffer ? chunk.toString() : String(chunk); + callback(); + }, + }); + return { stdout, read: () => captured }; +} + +describe("parseUserPromptSubmitPayload", () => { + it("parses valid JSON payload", async () => { + const raw = await readFile("test/fixtures/user-prompt-submit.json", "utf8"); + const parsed = parseUserPromptSubmitPayload(raw); + expect(parsed?.hook_event_name).toBe("UserPromptSubmit"); + expect(parsed?.prompt).toContain("OMO_ULW_LOOP_STEER"); + }); + + it("returns null for empty input", () => { + expect(parseUserPromptSubmitPayload("")).toBeNull(); + }); + + it("returns null for invalid JSON", () => { + expect(parseUserPromptSubmitPayload("{bad")).toBeNull(); + }); + + it("returns null when hook_event_name missing", () => { + expect(parseUserPromptSubmitPayload(JSON.stringify({ cwd: "/repo", prompt: "x", session_id: "s1" }))).toBeNull(); + }); +}); + +describe("applyUserPromptUlwLoopSteering - OMO directive patterns", () => { + it("processes OMO_ULW_LOOP_STEER: prompt and returns audit text on success", async () => { + const repoRoot = await bootstrapPlanRepo(); + const out = await applyUserPromptUlwLoopSteering( + payload( + 'OMO_ULW_LOOP_STEER: {"kind":"annotate_ledger","source":"user_prompt_submit","evidence":"x","rationale":"y"}', + repoRoot, + ), + ); + expect(out.length).toBeGreaterThan(0); + expect(out).toContain("annotate_ledger"); + }); + + it("#given a Codex session id #when steering from a hook #then writes the session-scoped ledger", async () => { + const repoRoot = await bootstrapPlanRepo(); + + const out = await applyUserPromptUlwLoopSteering( + payload( + 'OMO_ULW_LOOP_STEER: {"kind":"annotate_ledger","source":"user_prompt_submit","evidence":"x","rationale":"y"}', + repoRoot, + ), + ); + + expect(out).toContain("accepted"); + expect(await readFile(ulwLoopLedgerPath(repoRoot, { sessionId: DEFAULT_SESSION_ID }), "utf8")).toContain( + "steering_accepted", + ); + }); + + it("processes omo.ulw-loop.steer: pattern", async () => { + const repoRoot = await bootstrapPlanRepo(); + const out = await applyUserPromptUlwLoopSteering( + payload( + 'omo.ulw-loop.steer: {"kind":"annotate_ledger","source":"user_prompt_submit","evidence":"x","rationale":"y"}', + repoRoot, + ), + ); + expect(out).toContain("accepted"); + }); + + it("processes omo ulw-loop steer: pattern", async () => { + const repoRoot = await bootstrapPlanRepo(); + const out = await applyUserPromptUlwLoopSteering( + payload( + 'omo ulw-loop steer: {"kind":"annotate_ledger","source":"user_prompt_submit","evidence":"x","rationale":"y"}', + repoRoot, + ), + ); + expect(out).toContain("annotate_ledger"); + }); +}); + +describe("applyUserPromptUlwLoopSteering - non-matching prompts", () => { + it("returns empty string when no directive in prompt", async () => { + expect(await applyUserPromptUlwLoopSteering(payload("just a normal user message", "/tmp"))).toBe(""); + }); + + it("returns empty when hook_event_name is not UserPromptSubmit", async () => { + expect(await applyUserPromptUlwLoopSteering(payloadWithRuntimeEvent("PostToolUse"))).toBe(""); + }); +}); + +describe("applyUserPromptUlwLoopSteering - error swallowing", () => { + it("returns empty (never throws) when plan does not exist", async () => { + const repoRoot = await mkdtemp(join(tmpdir(), "ug-nohook-")); + const out = await applyUserPromptUlwLoopSteering( + payload( + 'OMO_ULW_LOOP_STEER: {"kind":"annotate_ledger","source":"user_prompt_submit","evidence":"x","rationale":"y"}', + repoRoot, + ), + ); + expect(out).toBe(""); + }); + + it("returns empty when steering proposal is malformed JSON after marker", async () => { + const out = await applyUserPromptUlwLoopSteering(payload("OMO_ULW_LOOP_STEER: {bad", "/tmp")); + expect(out).toBe(""); + }); +}); + +describe("runUlwLoopHookCli (stdin/stdout integration)", () => { + it("reads stdin, applies steering, writes audit to stdout", async () => { + const repoRoot = await bootstrapPlanRepo(); + const stdin = Readable.from([ + JSON.stringify( + payload( + 'OMO_ULW_LOOP_STEER: {"kind":"annotate_ledger","source":"user_prompt_submit","evidence":"x","rationale":"y"}', + repoRoot, + ), + ), + ]); + const capture = captureStdout(); + await runUlwLoopHookCli(stdin, capture.stdout); + expect(capture.read().length).toBeGreaterThan(0); + }); + + it("writes nothing when stdin is empty", async () => { + const capture = captureStdout(); + await runUlwLoopHookCli(Readable.from([""]), capture.stdout); + expect(capture.read()).toBe(""); + }); +}); + +describe("applyPreToolUseGoalBudgetGuard", () => { + it("#given create_goal sets token_budget #when PreToolUse runs #then it blocks with unlimited-goal warning", () => { + // given + const input = preToolPayload("create_goal", { objective: "Ship the feature", token_budget: 5000 }); + + // when + const output = applyPreToolUseGoalBudgetGuard(input); + + // then + const parsed = JSON.parse(output); + expect(parsed).toMatchObject({ + hookSpecificOutput: { + hookEventName: "PreToolUse", + permissionDecision: "deny", + }, + }); + expect(parsed.hookSpecificOutput.permissionDecisionReason).toContain("Do not set token_budget on create_goal"); + expect(parsed.hookSpecificOutput.permissionDecisionReason).toContain("unlimited"); + }); + + it("#given create_goal omits token_budget #when PreToolUse runs #then it stays silent", () => { + // given + const input = preToolPayload("create_goal", { objective: "Ship the feature" }); + + // when + const output = applyPreToolUseGoalBudgetGuard(input); + + // then + expect(output).toBe(""); + }); + + it("#given a neighboring tool includes token_budget text #when PreToolUse runs #then it stays silent", () => { + // given + const input = preToolPayload("update_goal", { status: "complete", token_budget: 5000 }); + + // when + const output = applyPreToolUseGoalBudgetGuard(input); + + // then + expect(output).toBe(""); + }); +}); + +describe("runPreToolUseGoalBudgetGuardCli", () => { + it("#given Codex PreToolUse stdin with budgeted create_goal #when CLI hook runs #then it writes blocking JSON", async () => { + // given + const stdin = Readable.from([ + JSON.stringify(preToolPayload("create_goal", { objective: "Ship", token_budget: 1 })), + ]); + const capture = captureStdout(); + + // when + await runPreToolUseGoalBudgetGuardCli(stdin, capture.stdout); + + // then + const parsed = JSON.parse(capture.read()); + expect(parsed.hookSpecificOutput.permissionDecision).toBe("deny"); + expect(parsed.hookSpecificOutput.permissionDecisionReason).toContain("unlimited"); + }); +}); diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/evidence-criteria-gate.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/evidence-criteria-gate.test.ts new file mode 100644 index 000000000..e02de24b4 --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/evidence-criteria-gate.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from "vitest"; + +import { requireAllCriteriaPass } from "../src/evidence.js"; +import type { UlwLoopItem, UlwLoopSuccessCriterion } from "../src/types.js"; +import { UlwLoopError } from "../src/types.js"; + +const NOW = "2026-05-23T00:00:00.000Z"; + +function makeCriterion(overrides: Partial = {}): UlwLoopSuccessCriterion { + return { + id: "C001", + scenario: "happy path login returns 200", + userModel: "happy", + expectedEvidence: "curl /login -d {valid} returns 200 + token", + capturedEvidence: null, + status: "pending", + ...overrides, + }; +} + +function makeGoal(overrides: Partial = {}): UlwLoopItem { + return { + id: "G001", + title: "Auth endpoint", + objective: "Build JWT auth", + status: "in_progress", + successCriteria: [ + makeCriterion({ id: "C001" }), + makeCriterion({ id: "C002", userModel: "edge" }), + makeCriterion({ id: "C003", userModel: "regression" }), + ], + attempt: 1, + createdAt: NOW, + updatedAt: NOW, + ...overrides, + }; +} + +describe("requireAllCriteriaPass", () => { + it("does NOT throw when all criteria pass", () => { + // given + const goal = makeGoal({ + successCriteria: [ + makeCriterion({ id: "C001", status: "pass" }), + makeCriterion({ id: "C002", status: "pass" }), + ], + }); + + // when / then + expect(() => requireAllCriteriaPass(goal)).not.toThrow(); + }); + + it("throws UlwLoopError when any criterion pending", () => { + // given + const goal = makeGoal({ + successCriteria: [ + makeCriterion({ id: "C001", status: "pass" }), + makeCriterion({ id: "C002", status: "pending" }), + makeCriterion({ id: "C003", status: "pass" }), + ], + }); + + // when / then + expect(() => requireAllCriteriaPass(goal)).toThrow(UlwLoopError); + }); + + it("throws when any fail/blocked too", () => { + // given + const goal1 = makeGoal({ successCriteria: [makeCriterion({ id: "C001", status: "fail" })] }); + const goal2 = makeGoal({ successCriteria: [makeCriterion({ id: "C001", status: "blocked" })] }); + + // when / then + expect(() => requireAllCriteriaPass(goal1)).toThrow(UlwLoopError); + expect(() => requireAllCriteriaPass(goal2)).toThrow(UlwLoopError); + }); + + it("UlwLoopError includes details.goalId + details.unresolved", () => { + // given + const goal = makeGoal({ + id: "G001", + successCriteria: [ + makeCriterion({ id: "C001", status: "pass" }), + makeCriterion({ id: "C002", status: "pending" }), + makeCriterion({ id: "C003", status: "pass" }), + ], + }); + + // when / then + try { + requireAllCriteriaPass(goal); + expect.fail("expected throw"); + } catch (error) { + expect(error).toBeInstanceOf(UlwLoopError); + if (!(error instanceof UlwLoopError)) throw error; + expect(error.code).toBe("ulw_loop_criteria_not_all_pass"); + expect(error.details?.["goalId"]).toBe("G001"); + expect(Array.isArray(error.details?.["unresolved"])).toBe(true); + } + }); +}); diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/evidence.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/evidence.test.ts new file mode 100644 index 000000000..061000f67 --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/evidence.test.ts @@ -0,0 +1,263 @@ +import { mkdir, mkdtemp, readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { + criteriaSummary, + markCriteriaPendingResetForGoal, + recordEvidence, + unresolvedCriteriaOf, +} from "../src/evidence.js"; +import { ulwLoopDir } from "../src/paths.js"; +import { readUlwLoopPlan, writePlan } from "../src/plan-io.js"; +import type { UlwLoopItem, UlwLoopLedgerEntry, UlwLoopPlan, UlwLoopSuccessCriterion } from "../src/types.js"; +import { UlwLoopError } from "../src/types.js"; + +const NOW = "2026-05-23T00:00:00.000Z"; + +async function bootstrapRepo(plan: UlwLoopPlan): Promise { + const repo = await mkdtemp(join(tmpdir(), "ug-evidence-")); + await mkdir(ulwLoopDir(repo), { recursive: true }); + await writePlan(repo, plan); + return repo; +} + +async function readLastLedgerEntry(repo: string): Promise { + const lines = (await readFile(join(repo, ".omo/ulw-loop/ledger.jsonl"), "utf8")).trim().split("\n"); + const last = lines.at(-1); + if (last === undefined) throw new Error("expected ledger entry"); + return JSON.parse(last); +} + +function firstGoal(plan: UlwLoopPlan): UlwLoopItem { + const goal = plan.goals.at(0); + if (goal === undefined) throw new Error("expected goal"); + return goal; +} + +function makeCriterion(overrides: Partial = {}): UlwLoopSuccessCriterion { + return { + id: "C001", + scenario: "happy path login returns 200", + userModel: "happy", + expectedEvidence: "curl /login -d {valid} returns 200 + token", + capturedEvidence: null, + status: "pending", + ...overrides, + }; +} + +function makeGoal(overrides: Partial = {}): UlwLoopItem { + return { + id: "G001", + title: "Auth endpoint", + objective: "Build JWT auth", + status: "in_progress", + successCriteria: [ + makeCriterion({ id: "C001" }), + makeCriterion({ id: "C002", userModel: "edge" }), + makeCriterion({ id: "C003", userModel: "regression" }), + ], + attempt: 1, + createdAt: NOW, + updatedAt: NOW, + ...overrides, + }; +} + +function makePlan(overrides: Partial = {}): UlwLoopPlan { + return { + version: 1, + createdAt: NOW, + updatedAt: NOW, + briefPath: ".omo/ulw-loop/brief.md", + goalsPath: ".omo/ulw-loop/goals.json", + ledgerPath: ".omo/ulw-loop/ledger.jsonl", + codexGoalMode: "aggregate", + codexObjective: "Complete the durable ulw-loop plan in .omo/ulw-loop/goals.json", + codexObjectiveAliases: [], + goals: [makeGoal()], + ...overrides, + }; +} + +describe("recordEvidence (status=pass)", () => { + it("sets criterion.status=pass + capturedEvidence + capturedAt", async () => { + const repo = await bootstrapRepo(makePlan()); + + const result = await recordEvidence(repo, { + goalId: "G001", + criterionId: "C001", + status: "pass", + evidence: "curl /login returns 200 + token verified", + }); + + expect(result.criterion.status).toBe("pass"); + expect(result.criterion.capturedEvidence).toContain("curl /login returns 200"); + expect(result.criterion.capturedAt).toMatch(/^\d{4}-\d{2}-\d{2}T/); + }); + + it("appends evidence_captured ledger event", async () => { + const repo = await bootstrapRepo(makePlan()); + + await recordEvidence(repo, { goalId: "G001", criterionId: "C001", status: "pass", evidence: "observable proof" }); + + const last = await readLastLedgerEntry(repo); + expect(last.kind).toBe("evidence_captured"); + expect(last.goalId).toBe("G001"); + expect(last.criterionId).toBe("C001"); + }); + + it("persists the change so a fresh read sees status=pass", async () => { + const repo = await bootstrapRepo(makePlan()); + + await recordEvidence(repo, { goalId: "G001", criterionId: "C001", status: "pass", evidence: "observable proof" }); + + const criterion = firstGoal(await readUlwLoopPlan(repo)).successCriteria.find((c) => c.id === "C001"); + expect(criterion?.status).toBe("pass"); + }); +}); + +describe("recordEvidence (status=fail)", () => { + it("sets criterion.status=fail + appends criterion_failed event", async () => { + const repo = await bootstrapRepo(makePlan()); + + const result = await recordEvidence(repo, { + goalId: "G001", + criterionId: "C001", + status: "fail", + evidence: "got 500 not 200", + }); + + expect(result.criterion.status).toBe("fail"); + expect((await readLastLedgerEntry(repo)).kind).toBe("criterion_failed"); + }); +}); + +describe("recordEvidence (status=blocked)", () => { + it("sets criterion.status=blocked + appends criterion_blocked event", async () => { + const repo = await bootstrapRepo(makePlan()); + + const result = await recordEvidence(repo, { + goalId: "G001", + criterionId: "C001", + status: "blocked", + evidence: "auth not in CI yet", + }); + + expect(result.criterion.status).toBe("blocked"); + expect((await readLastLedgerEntry(repo)).kind).toBe("criterion_blocked"); + }); +}); + +describe("recordEvidence error cases", () => { + it("throws when goalId not found", async () => { + const repo = await bootstrapRepo(makePlan()); + + await expect( + recordEvidence(repo, { goalId: "GUNKNOWN", criterionId: "C001", status: "pass", evidence: "x" }), + ).rejects.toBeInstanceOf(UlwLoopError); + }); + + it("throws when criterionId not found within goal", async () => { + const repo = await bootstrapRepo(makePlan()); + + await expect( + recordEvidence(repo, { goalId: "G001", criterionId: "CUNKNOWN", status: "pass", evidence: "x" }), + ).rejects.toBeInstanceOf(UlwLoopError); + }); + + it("throws when evidence is empty/whitespace", async () => { + const repo = await bootstrapRepo(makePlan()); + + await expect( + recordEvidence(repo, { goalId: "G001", criterionId: "C001", status: "pass", evidence: " " }), + ).rejects.toBeInstanceOf(UlwLoopError); + }); +}); + +describe("markCriteriaPendingResetForGoal", () => { + it("resets every criterion of the goal to pending + capturedEvidence=null", async () => { + const goal = makeGoal({ + successCriteria: [ + makeCriterion({ id: "C001", status: "pass", capturedEvidence: "old" }), + makeCriterion({ id: "C002", status: "fail", capturedEvidence: "older" }), + makeCriterion({ id: "C003", status: "blocked", capturedEvidence: "oldest" }), + ], + }); + const repo = await bootstrapRepo(makePlan({ goals: [goal] })); + + const result = await markCriteriaPendingResetForGoal(repo, "G001"); + + expect(result.resetCount).toBe(3); + for (const c of firstGoal(result.plan).successCriteria) { + expect(c.status).toBe("pending"); + expect(c.capturedEvidence).toBeNull(); + } + }); + + it("appends a single criteria_revised ledger event describing the reset", async () => { + const repo = await bootstrapRepo(makePlan()); + + await markCriteriaPendingResetForGoal(repo, "G001"); + + expect((await readLastLedgerEntry(repo)).kind).toBe("criteria_revised"); + }); +}); + +describe("criteriaSummary (pure)", () => { + it("aggregates counts across all goals", () => { + const plan = makePlan({ + goals: [ + makeGoal({ + id: "G001", + successCriteria: [ + makeCriterion({ id: "C001", status: "pass" }), + makeCriterion({ id: "C002", status: "pending" }), + ], + }), + makeGoal({ + id: "G002", + successCriteria: [ + makeCriterion({ id: "C001", status: "fail" }), + makeCriterion({ id: "C002", status: "blocked" }), + makeCriterion({ id: "C003", status: "pass" }), + ], + }), + ], + }); + + const summary = criteriaSummary(plan); + + expect(summary.totalCriteria).toBe(5); + expect(summary.passCount).toBe(2); + expect(summary.pendingCount).toBe(1); + expect(summary.failCount).toBe(1); + expect(summary.blockedCount).toBe(1); + expect(summary.goalsWithUnresolvedCriteria).toEqual(["G001", "G002"]); + }); + + it("returns empty when no criteria exist", () => { + const summary = criteriaSummary(makePlan({ goals: [makeGoal({ successCriteria: [] })] })); + + expect(summary.totalCriteria).toBe(0); + expect(summary.goalsWithUnresolvedCriteria).toEqual([]); + }); +}); + +describe("unresolvedCriteriaOf (pure)", () => { + it("returns only non-pass criteria", () => { + const goal = makeGoal({ + successCriteria: [ + makeCriterion({ id: "C001", status: "pass" }), + makeCriterion({ id: "C002", status: "pending" }), + makeCriterion({ id: "C003", status: "fail" }), + ], + }); + + const unresolved = unresolvedCriteriaOf(goal); + + expect(unresolved.map((c) => c.id)).toEqual(["C002", "C003"]); + }); +}); diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/goal-status.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/goal-status.test.ts new file mode 100644 index 000000000..04670c7c6 --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/goal-status.test.ts @@ -0,0 +1,327 @@ +import { describe, expect, it } from "vitest"; + +import { + aggregateCodexObjective, + codexGoalMode, + compatibleCodexObjectives, + expectedCodexObjective, + firstUnresolvedCriterion, + hasAllCriteriaPass, + isFinalRunCompletionCandidate, + isUlwLoopDone, + ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE, +} from "../src/goal-status.js"; +import type { UlwLoopItem, UlwLoopPlan, UlwLoopSuccessCriterion } from "../src/types.js"; + +const NOW = "2026-05-23T00:00:00.000Z"; + +function makeCriterion(overrides: Partial = {}): UlwLoopSuccessCriterion { + return { + id: "C001", + scenario: "happy path", + userModel: "happy", + expectedEvidence: "observable proof", + capturedEvidence: null, + status: "pending", + ...overrides, + }; +} + +function makeGoal(overrides: Partial = {}): UlwLoopItem { + return { + id: "G001", + title: "Goal one", + objective: "Complete goal one", + status: "pending", + successCriteria: [], + attempt: 1, + createdAt: NOW, + updatedAt: NOW, + ...overrides, + }; +} + +function makePlan(overrides: Partial = {}): UlwLoopPlan { + return { + version: 1, + createdAt: NOW, + updatedAt: NOW, + briefPath: ".omo/ulw-loop/brief.md", + goalsPath: ".omo/ulw-loop/goals.json", + ledgerPath: ".omo/ulw-loop/ledger.jsonl", + goals: [], + ...overrides, + }; +} + +describe("isUlwLoopDone", () => { + it("returns true when all goals complete", () => { + // given + const plan = makePlan({ + goals: [makeGoal({ status: "complete" }), makeGoal({ id: "G002", status: "complete" })], + }); + + // when + const done = isUlwLoopDone(plan); + + // then + expect(done).toBe(true); + }); + + it("returns false when any pending remains", () => { + // given + const plan = makePlan({ goals: [makeGoal({ status: "complete" }), makeGoal({ id: "G002", status: "pending" })] }); + + // when + const done = isUlwLoopDone(plan); + + // then + expect(done).toBe(false); + }); + + it("treats superseded-with-complete-replacements as resolved", () => { + // given + const replacement = makeGoal({ id: "G002", status: "complete" }); + const superseded = makeGoal({ + id: "G001", + status: "pending", + steeringStatus: "superseded", + supersededBy: [replacement.id], + }); + const plan = makePlan({ goals: [superseded, replacement] }); + + // when + const done = isUlwLoopDone(plan); + + // then + expect(done).toBe(true); + }); +}); + +describe("isFinalRunCompletionCandidate", () => { + it("returns true when only one unresolved goal remains", () => { + // given + const finalGoal = makeGoal({ id: "G002", status: "pending" }); + const plan = makePlan({ goals: [makeGoal({ status: "complete" }), finalGoal] }); + + // when + const candidate = isFinalRunCompletionCandidate(plan, finalGoal); + + // then + expect(candidate).toBe(true); + }); + + it("returns false when multiple unresolved", () => { + // given + const goal = makeGoal({ id: "G001", status: "pending" }); + const plan = makePlan({ goals: [goal, makeGoal({ id: "G002", status: "pending" })] }); + + // when + const candidate = isFinalRunCompletionCandidate(plan, goal); + + // then + expect(candidate).toBe(false); + }); +}); + +describe("codexGoalMode", () => { + it("defaults to per_story when undefined", () => { + // when + const mode = codexGoalMode(makePlan()); + + // then + expect(mode).toBe("per_story"); + }); + + it("returns aggregate when explicitly aggregate", () => { + // when + const mode = codexGoalMode(makePlan({ codexGoalMode: "aggregate" })); + + // then + expect(mode).toBe("aggregate"); + }); +}); + +describe("expectedCodexObjective", () => { + it("aggregate mode returns plan.codexObjective", () => { + // given + const goal = makeGoal({ objective: "story objective" }); + const plan = makePlan({ codexGoalMode: "aggregate", codexObjective: "aggregate objective" }); + + // when + const objective = expectedCodexObjective(plan, goal); + + // then + expect(objective).toBe("aggregate objective"); + }); + + it("aggregate mode falls back to ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE when codexObjective missing", () => { + // given + const goal = makeGoal({ objective: "story objective" }); + const plan = makePlan({ codexGoalMode: "aggregate" }); + + // when + const objective = expectedCodexObjective(plan, goal); + + // then + expect(objective).toBe(ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE); + }); + + it("per_story mode returns goal.objective", () => { + // given + const goal = makeGoal({ objective: "story objective" }); + const plan = makePlan({ codexGoalMode: "per_story", codexObjective: "aggregate objective" }); + + // when + const objective = expectedCodexObjective(plan, goal); + + // then + expect(objective).toBe("story objective"); + }); +}); + +describe("aggregateCodexObjective", () => { + it("returns plan.codexObjective when set", () => { + // when + const objective = aggregateCodexObjective(makePlan({ codexObjective: "aggregate objective" })); + + // then + expect(objective).toBe("aggregate objective"); + }); + + it("falls back to ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE", () => { + // when + const objective = aggregateCodexObjective(makePlan()); + + // then + expect(objective).toBe(ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE); + }); +}); + +describe("compatibleCodexObjectives", () => { + it("includes aggregate objective + aliases", () => { + // given + const plan = makePlan({ + codexObjective: "aggregate objective", + codexObjectiveAliases: ["legacy one", "legacy two"], + }); + + // when + const objectives = compatibleCodexObjectives(plan); + + // then + expect(objectives).toEqual(["aggregate objective", "legacy one", "legacy two"]); + }); +}); + +describe("hasAllCriteriaPass", () => { + it("returns true when all criteria pass", () => { + // given + const goal = makeGoal({ + successCriteria: [makeCriterion({ status: "pass" }), makeCriterion({ id: "C002", status: "pass" })], + }); + + // when + const passed = hasAllCriteriaPass(goal); + + // then + expect(passed).toBe(true); + }); + + it("returns false when any criterion pending", () => { + // given + const goal = makeGoal({ + successCriteria: [makeCriterion({ status: "pass" }), makeCriterion({ id: "C002", status: "pending" })], + }); + + // when + const passed = hasAllCriteriaPass(goal); + + // then + expect(passed).toBe(false); + }); + + it("returns false when any criterion fail", () => { + // given + const goal = makeGoal({ + successCriteria: [makeCriterion({ status: "pass" }), makeCriterion({ id: "C002", status: "fail" })], + }); + + // when + const passed = hasAllCriteriaPass(goal); + + // then + expect(passed).toBe(false); + }); + + it("returns false when any criterion blocked", () => { + // given + const goal = makeGoal({ + successCriteria: [makeCriterion({ status: "pass" }), makeCriterion({ id: "C002", status: "blocked" })], + }); + + // when + const passed = hasAllCriteriaPass(goal); + + // then + expect(passed).toBe(false); + }); + + it("returns false for empty criteria array", () => { + // when + const passed = hasAllCriteriaPass(makeGoal({ successCriteria: [] })); + + // then + expect(passed).toBe(false); + }); +}); + +describe("firstUnresolvedCriterion", () => { + it("returns first non-pass criterion", () => { + // given + const unresolved = makeCriterion({ id: "C002", status: "fail" }); + const goal = makeGoal({ successCriteria: [makeCriterion({ status: "pass" }), unresolved] }); + + // when + const criterion = firstUnresolvedCriterion(goal); + + // then + expect(criterion).toBe(unresolved); + }); + + it("returns undefined when all pass", () => { + // given + const goal = makeGoal({ + successCriteria: [makeCriterion({ status: "pass" }), makeCriterion({ id: "C002", status: "pass" })], + }); + + // when + const criterion = firstUnresolvedCriterion(goal); + + // then + expect(criterion).toBeUndefined(); + }); + + it("returns first pending in mixed pass/pending/fail", () => { + // given + const pending = makeCriterion({ id: "C002", status: "pending" }); + const goal = makeGoal({ + successCriteria: [makeCriterion({ status: "pass" }), pending, makeCriterion({ id: "C003", status: "fail" })], + }); + + // when + const criterion = firstUnresolvedCriterion(goal); + + // then + expect(criterion).toBe(pending); + }); +}); + +describe("ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE", () => { + it("references the .omo/ulw-loop path and excludes the legacy workspace", () => { + const legacyWorkspace = [".", "om", "x"].join(""); + + expect(ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE).toContain(".omo/ulw-loop"); + expect(ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE).not.toContain(legacyWorkspace); + }); +}); diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/package-smoke.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/package-smoke.test.ts new file mode 100644 index 000000000..995cc290d --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/package-smoke.test.ts @@ -0,0 +1,150 @@ +// biome-ignore-all format: smoke test pulls verbatim JSON for structural assertion. +import { readFile, stat } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +async function readText(relative: string): Promise { + return readFile(join(repoRoot, relative), "utf8"); +} + +async function readJson(relative: string): Promise { + return JSON.parse(await readText(relative)); +} + +describe("package.json", () => { + it("declares ESM + npm + Node >=20", async () => { + const pkg = await readJson("package.json") as Record; + expect(pkg["type"]).toBe("module"); + expect(pkg["packageManager"]).toBe("npm@11.12.1"); + expect((pkg["engines"] as Record)["node"]).toBe(">=20.0.0"); + }); + + it("exposes the omo binary pointing at dist/cli.js", async () => { + const pkg = await readJson("package.json") as Record; + const bin = pkg["bin"] as Record; + expect(bin["omo"]).toBe("./dist/cli.js"); + }); + + it("ships the expected files for npm publish", async () => { + const pkg = await readJson("package.json") as Record; + const files = pkg["files"] as readonly string[]; + expect(files).toContain("dist"); + expect(files).toContain("hooks"); + expect(files).toContain("skills"); + expect(files).not.toContain(".codex-plugin"); + }); +}); + +describe("component plugin identity", () => { + it("is owned by the aggregate OMO plugin root", async () => { + await expect(readText(".codex-plugin/plugin.json")).rejects.toMatchObject({ code: "ENOENT" }); + }); +}); + +describe("hooks/hooks.json", () => { + it("registers UserPromptSubmit with PLUGIN_ROOT interpolation", async () => { + const hooks = await readJson("hooks/hooks.json") as Record; + const events = (hooks["hooks"] as Record)["UserPromptSubmit"] as readonly Record[]; + expect(events.length).toBeGreaterThan(0); + const command = ((events[0]?.["hooks"] as readonly Record[])[0]?.["command"]) as string; + expect(command).toContain(`$${"{PLUGIN_ROOT}"}`); + expect(command).toContain("dist/cli.js"); + expect(command).toContain("hook user-prompt-submit"); + }); + + it("#given ulw-loop component is enabled #when hooks are inspected #then create_goal PreToolUse guard is registered", async () => { + const text = await readText("hooks/hooks.json"); + + expect(text).toContain('"PreToolUse"'); + expect(text).toContain('"matcher": "^create_goal$"'); + expect(text).toContain("hook pre-tool-use"); + }); +}); + +describe("src/cli.ts", () => { + it("starts with #!/usr/bin/env node shebang", async () => { + const text = await readText("src/cli.ts"); + expect(text.split("\n")[0]).toBe("#!/usr/bin/env node"); + }); +}); + +describe("skills/ulw-loop/SKILL.md", () => { + it("exists", async () => { + const info = await stat(join(repoRoot, "skills/ulw-loop/SKILL.md")); + expect(info.isFile()).toBe(true); + }); + + it("#given Codex skill hinting #when ulw-loop skill metadata is inspected #then ulw-loop is the primary mention name", async () => { + const text = await readText("skills/ulw-loop/SKILL.md"); + + expect(text).toMatch(/^---\nname: ulw-loop\n/m); + expect(text).toContain("Goal-like loop that uses ultrawork mode to decompose work into systematic, evidence-bound steps."); + expect(text).toContain("short-description: Goal-like ultrawork loop for systematic decomposition"); + }); + + it("#given Codex dollar hinting #when querying ulw-loop #then ulw-loop surfaces the ulw-loop alias", async () => { + const text = await readText("skills/ulw-loop/agents/openai.yaml"); + + expect(text).toContain('display_name: "ulw loop"'); + expect(text).not.toContain("ulw-loop / ulw-loop"); + expect(text).toContain('short_description: "Goal-like ultrawork loop for systematic decomposition"'); + expect(text).toContain("Use $ulw-loop"); + }); + + it("#given Codex dollar hinting #when querying ulw-loop #then ulw-loop remains discoverable as an alias", async () => { + const text = await readText("skills/ulw-loop/agents/openai.yaml"); + + expect(text).toContain("search_terms:"); + expect(text).toContain('- "ulw-loop"'); + }); + + it("references the success criteria and record-evidence vocabulary", async () => { + const text = await readText("skills/ulw-loop/SKILL.md"); + expect(text.toLowerCase()).toMatch(/success criteria|successcriteria/); + expect(text.toLowerCase()).toContain("record-evidence"); + }); + + it("#given omo is absent from PATH #when bootstrap instructions are read #then local cached CLI fallback is documented", async () => { + const text = await readText("skills/ulw-loop/SKILL.md"); + + expect(text).toContain("If `omo` is absent from PATH"); + expect(text).toContain("ULW_LOOP_CLI"); + expect(text).toContain("components/ulw-loop/dist/cli.js"); + }); + + it("#given empty PATH #when bootstrap instructions are read #then handles empty PATH without losing notepad bootstrap", async () => { + const text = await readText("skills/ulw-loop/SKILL.md"); + + expect(text).toContain("If PATH is empty"); + expect(text).toContain("ULW_LOOP_NODE"); + expect(text).toContain(".omo/ulw-loop/bootstrap-notepad.md"); + expect(text).not.toContain("ls -1"); + }); + + it("uses the .omo workspace path", async () => { + const text = await readText("skills/ulw-loop/SKILL.md"); + expect(text).toContain(".omo/ulw-loop"); + }); +}); + +describe("source LOC budget", () => { + it("every source file stays at or under 250 pure LOC", async () => { + const files = [ + "src/types.ts", "src/paths.ts", "src/plan-io.ts", "src/plan-crud.ts", "src/goal-status.ts", + "src/evidence.ts", "src/quality-gate.ts", "src/checkpoint.ts", "src/review-blockers.ts", + "src/steering.ts", "src/codex-goal-instruction.ts", "src/codex-goal-snapshot.ts", "src/codex-hook.ts", + "src/cli.ts", "src/cli-arg-parser.ts", "src/cli-output.ts", "src/cli-steering.ts", "src/cli-commands.ts", + ]; + for (const file of files) { + const text = await readText(file); + const pure = text.split("\n").filter((line) => { + const trimmed = line.trim(); + return trimmed.length > 0 && !trimmed.startsWith("//"); + }).length; + expect(pure, `${file} pure LOC`).toBeLessThanOrEqual(250); + } + }); +}); diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/paths.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/paths.test.ts new file mode 100644 index 000000000..be781f92d --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/paths.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; + +import { + normalizeUlwLoopSessionId, + repoRelative, + ulwLoopBriefPath, + ulwLoopDir, + ulwLoopGoalsPath, + ulwLoopLedgerPath, +} from "../src/paths.ts"; + +describe("ulwLoopDir(repo)", () => { + it("returns repo + '/.omo/ulw-loop'", () => { + // when/then + expect(ulwLoopDir("/repo")).toBe("/repo/.omo/ulw-loop"); + }); + + it("#given a session id #when resolving the loop dir #then scopes artifacts under that session", () => { + // when/then + expect(ulwLoopDir("/repo", { sessionId: "sess_abc" })).toBe("/repo/.omo/ulw-loop/sess_abc"); + }); +}); + +describe("ulw-loop*Path helpers", () => { + it("compose artifact filenames under ulwLoopDir", () => { + // when/then + expect(ulwLoopBriefPath("/r")).toBe("/r/.omo/ulw-loop/brief.md"); + expect(ulwLoopGoalsPath("/r")).toBe("/r/.omo/ulw-loop/goals.json"); + expect(ulwLoopLedgerPath("/r")).toBe("/r/.omo/ulw-loop/ledger.jsonl"); + }); + + it("#given a session id #when composing artifact filenames #then returns session-scoped paths", () => { + // when/then + expect(ulwLoopBriefPath("/r", { sessionId: "session-A" })).toBe("/r/.omo/ulw-loop/session-A/brief.md"); + expect(ulwLoopGoalsPath("/r", { sessionId: "session-A" })).toBe("/r/.omo/ulw-loop/session-A/goals.json"); + expect(ulwLoopLedgerPath("/r", { sessionId: "session-A" })).toBe("/r/.omo/ulw-loop/session-A/ledger.jsonl"); + }); +}); + +describe("normalizeUlwLoopSessionId", () => { + it("#given traversal-like input #when normalized #then returns a path-safe session segment", () => { + // when/then + expect(normalizeUlwLoopSessionId("../bad/id")).toBe("bad-id"); + }); + + it("#given blank input #when normalized #then returns null", () => { + // when/then + expect(normalizeUlwLoopSessionId(" ")).toBeNull(); + }); +}); + +describe("repoRelative", () => { + it("strips repo prefix when path is inside repo", () => { + // when/then + expect(repoRelative("/repo/.omo/ulw-loop/goals.json", "/repo")).toBe(".omo/ulw-loop/goals.json"); + }); + + it("returns absolute when path is outside repo", () => { + // when/then + expect(repoRelative("/elsewhere/file", "/repo")).toBe("/elsewhere/file"); + }); +}); diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/plan-crud.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/plan-crud.test.ts new file mode 100644 index 000000000..abc80dcf2 --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/plan-crud.test.ts @@ -0,0 +1,256 @@ +import { mkdir, mkdtemp, readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { ulwLoopBriefPath, ulwLoopGoalsPath, ulwLoopLedgerPath } from "../src/paths.js"; +import { + addUlwLoopGoal, + createUlwLoopPlan, + deriveGoalCandidates, + seedDefaultSuccessCriteria, + startNextUlwLoop, + summarizeUlwLoopPlan, +} from "../src/plan-crud.js"; +import { writePlan } from "../src/plan-io.js"; +import type { UlwLoopItem, UlwLoopPlan, UlwLoopSuccessCriterion } from "../src/types.js"; +import { UlwLoopError } from "../src/types.js"; + +const NOW = "2026-05-23T00:00:00.000Z"; + +async function makeRepo(): Promise { + return mkdtemp(join(tmpdir(), "ug-crud-")); +} + +async function readBriefFixture(): Promise { + return readFile(join(process.cwd(), "test", "fixtures", "sample-brief.md"), "utf8"); +} + +async function ledgerKinds(repoRoot: string): Promise { + const raw = await readFile(ulwLoopLedgerPath(repoRoot), "utf8"); + return raw + .split(/\r?\n/) + .filter(Boolean) + .map((line) => JSON.parse(line).kind); +} + +function criterion(status: UlwLoopSuccessCriterion["status"]): UlwLoopSuccessCriterion { + const [base] = seedDefaultSuccessCriteria(0, "Implement auth endpoint"); + if (base === undefined) throw new Error("expected seeded criterion"); + return { ...base, status }; +} + +function makeGoal(overrides: Partial = {}): UlwLoopItem { + return { + id: "G001", + title: "Build auth service", + objective: "Implement JWT auth endpoint", + status: "pending", + successCriteria: seedDefaultSuccessCriteria(0, "Implement JWT auth endpoint"), + attempt: 0, + createdAt: NOW, + updatedAt: NOW, + ...overrides, + }; +} + +function makePlan(goals: UlwLoopItem[]): UlwLoopPlan { + return { + version: 1, + createdAt: NOW, + updatedAt: NOW, + briefPath: ".omo/ulw-loop/brief.md", + goalsPath: ".omo/ulw-loop/goals.json", + ledgerPath: ".omo/ulw-loop/ledger.jsonl", + codexGoalMode: "aggregate", + goals, + }; +} + +function scheduled(result: Awaited>) { + if ("done" in result) throw new Error("expected scheduled goal"); + return result; +} + +describe("seedDefaultSuccessCriteria", () => { + it("produces 3 criteria with C001/C002/C003 ids", () => { + const cs = seedDefaultSuccessCriteria(0, "Implement auth endpoint"); + expect(cs).toHaveLength(3); + expect(cs.map((c) => c.id)).toEqual(["C001", "C002", "C003"]); + }); + + it("covers happy + edge + regression user models", () => { + const cs = seedDefaultSuccessCriteria(0, "Implement auth endpoint"); + expect(cs.map((c) => c.userModel).sort()).toEqual(["edge", "happy", "regression"]); + }); + + it("seeds all criteria as pending with null capturedEvidence", () => { + const cs = seedDefaultSuccessCriteria(0, "Implement auth endpoint"); + for (const c of cs) { + expect(c.status).toBe("pending"); + expect(c.capturedEvidence).toBeNull(); + } + }); +}); + +describe("createUlwLoopPlan", () => { + it("creates .omo/ulw-loop/{brief.md, goals.json, ledger.jsonl} in repoRoot", async () => { + const repoRoot = await makeRepo(); + const brief = await readBriefFixture(); + + await createUlwLoopPlan(repoRoot, { brief }); + + expect(await readFile(ulwLoopBriefPath(repoRoot), "utf8")).toBe(brief.endsWith("\n") ? brief : `${brief}\n`); + expect(await readFile(ulwLoopGoalsPath(repoRoot), "utf8")).toContain("G001-build-the-jwt-auth-endpoint"); + expect(await ledgerKinds(repoRoot)).toEqual(["plan_created"]); + }); + + it("seeds at least 3 successCriteria per goal", async () => { + const plan = await createUlwLoopPlan(await makeRepo(), { brief: await readBriefFixture() }); + + expect(plan.goals).toHaveLength(3); + expect(plan.goals.every((goal) => goal.successCriteria.length >= 3)).toBe(true); + }); + + it("refuses overwrite of an existing plan without --force", async () => { + const repoRoot = await makeRepo(); + await createUlwLoopPlan(repoRoot, { brief: "first" }); + + await expect(createUlwLoopPlan(repoRoot, { brief: "second" })).rejects.toThrow(UlwLoopError); + await expect(createUlwLoopPlan(repoRoot, { brief: "second" })).rejects.toThrow("Refusing to overwrite"); + }); + + it("aggregate is the default codexGoalMode", async () => { + const plan = await createUlwLoopPlan(await makeRepo(), { brief: "Ship the feature" }); + + expect(plan.codexGoalMode).toBe("aggregate"); + expect(plan.codexObjective).toContain(".omo/ulw-loop/goals.json"); + }); +}); + +describe("deriveGoalCandidates", () => { + it("extracts bullets as goals", () => { + expect(deriveGoalCandidates("# Brief\n\n- Build auth\n- Add tests")).toEqual([ + { title: "Build auth", objective: "Build auth" }, + { title: "Add tests", objective: "Add tests" }, + ]); + }); + + it("falls back to paragraph parsing when no bullets", () => { + expect(deriveGoalCandidates("First objective.\n\nSecond objective.").map((goal) => goal.objective)).toEqual([ + "First objective.", + "Second objective.", + ]); + }); + + it("returns single default goal for empty/whitespace brief", () => { + expect(deriveGoalCandidates(" \n\t ")).toEqual([ + { title: "Complete the requested project objective.", objective: "Complete the requested project objective." }, + ]); + }); +}); + +describe("addUlwLoopGoal", () => { + it("appends a new goal to plan with seeded successCriteria", async () => { + const repoRoot = await makeRepo(); + await createUlwLoopPlan(repoRoot, { brief: "Build auth" }); + + const { plan, goal } = await addUlwLoopGoal(repoRoot, { title: "Add rate limit", objective: "Throttle login" }); + + expect(plan.goals).toHaveLength(2); + expect(goal.id).toBe("G002-add-rate-limit"); + expect(goal.successCriteria).toHaveLength(3); + }); + + it("appends a ledger entry for goal_added", async () => { + const repoRoot = await makeRepo(); + await createUlwLoopPlan(repoRoot, { brief: "Build auth" }); + + await addUlwLoopGoal(repoRoot, { title: "Add rate limit", objective: "Throttle login" }); + + expect(await ledgerKinds(repoRoot)).toEqual(["plan_created", "goal_added"]); + }); +}); + +describe("startNextUlwLoop", () => { + it("picks the first pending goal", async () => { + const repoRoot = await makeRepo(); + await createUlwLoopPlan(repoRoot, { brief: "- First\n- Second" }); + + const result = scheduled(await startNextUlwLoop(repoRoot, {})); + + expect(result.goal.id).toBe("G001-first"); + expect(result.goal.status).toBe("in_progress"); + expect(result.resumed).toBe(false); + }); + + it("resumes the in_progress goal when one exists", async () => { + const repoRoot = await makeRepo(); + const plan = await createUlwLoopPlan(repoRoot, { brief: "- First\n- Second" }); + const active = makeGoal({ ...plan.goals[1], status: "in_progress" }); + await writePlan(repoRoot, { ...plan, goals: [makeGoal({ ...plan.goals[0] }), active], activeGoalId: active.id }); + + const result = scheduled(await startNextUlwLoop(repoRoot, {})); + + expect(result.goal.id).toBe(active.id); + expect(result.resumed).toBe(true); + }); + + it("with retryFailed picks first failed (non-blocked) goal", async () => { + const repoRoot = await makeRepo(); + const failed = makeGoal({ status: "failed", failureReason: "flake" }); + await mkdir(join(repoRoot, ".omo", "ulw-loop"), { recursive: true }); + await writePlan(repoRoot, makePlan([failed])); + + const result = scheduled(await startNextUlwLoop(repoRoot, { retryFailed: true })); + + expect(result.goal.id).toBe("G001"); + expect(result.goal.attempt).toBe(1); + expect(await ledgerKinds(repoRoot)).toEqual(["goal_retried", "goal_started"]); + }); + + it("returns { done: true } when no eligible goals remain", async () => { + const repoRoot = await makeRepo(); + await mkdir(join(repoRoot, ".omo", "ulw-loop"), { recursive: true }); + await writePlan(repoRoot, makePlan([makeGoal({ status: "complete" })])); + + const result = await startNextUlwLoop(repoRoot, {}); + + expect(result).toMatchObject({ done: true }); + }); +}); + +describe("summarizeUlwLoopPlan", () => { + it("counts goals by status", () => { + const plan = makePlan([ + makeGoal({ id: "G001", status: "pending" }), + makeGoal({ id: "G002", status: "in_progress" }), + makeGoal({ id: "G003", status: "complete" }), + makeGoal({ id: "G004", status: "failed" }), + makeGoal({ id: "G005", status: "blocked", steeringStatus: "blocked" }), + makeGoal({ id: "G006", status: "review_blocked" }), + makeGoal({ id: "G007", status: "needs_user_decision", steeringStatus: "superseded" }), + ]); + + expect(summarizeUlwLoopPlan(plan)).toMatchObject({ + total: 7, + pending: 1, + in_progress: 1, + complete: 1, + failed: 1, + blocked: 1, + review_blocked: 1, + needs_user_decision: 1, + superseded: 1, + }); + }); + + it("aggregates criteria pass/pending/fail/blocked across all goals", () => { + const plan = makePlan([ + makeGoal({ successCriteria: [criterion("pass"), criterion("pending")] }), + makeGoal({ id: "G002", successCriteria: [criterion("fail"), criterion("blocked"), criterion("pending")] }), + ]); + + expect(summarizeUlwLoopPlan(plan).criteria).toEqual({ total: 5, pass: 1, pending: 2, fail: 1, blocked: 1 }); + }); +}); diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/plan-io.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/plan-io.test.ts new file mode 100644 index 000000000..fe4a07775 --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/plan-io.test.ts @@ -0,0 +1,239 @@ +import { copyFile, mkdir, mkdtemp, readdir, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { beforeEach, describe, expect, it } from "vitest"; +import { ulwLoopDir, ulwLoopGoalsPath, ulwLoopLedgerPath } from "../src/paths.js"; +import { + appendLedger, + readSteeringLedgerEntries, + readUlwLoopPlan, + withUlwLoopMutationLock, + writePlan, +} from "../src/plan-io.js"; +import type { UlwLoopItem, UlwLoopLedgerEntry, UlwLoopPlan } from "../src/types.js"; +import { UlwLoopError } from "../src/types.js"; + +const NOW = "2026-05-23T00:00:00.000Z"; +const STABLE_OBJECTIVE = + "Complete the durable ulw-loop plan in .omo/ulw-loop/goals.json, including later accepted/appended stories, under the original brief constraints; use .omo/ulw-loop/ledger.jsonl as the audit trail."; + +function makeGoal(overrides: Partial = {}): UlwLoopItem { + return { + id: "G001", + title: "Build auth service", + objective: "Implement JWT auth endpoint", + status: "pending", + successCriteria: [], + attempt: 1, + createdAt: NOW, + updatedAt: NOW, + ...overrides, + }; +} + +function makePlan(overrides: Partial = {}): UlwLoopPlan { + return { + version: 1, + createdAt: NOW, + updatedAt: NOW, + briefPath: ".omo/ulw-loop/brief.md", + goalsPath: ".omo/ulw-loop/goals.json", + ledgerPath: ".omo/ulw-loop/ledger.jsonl", + codexGoalMode: "aggregate", + codexObjective: STABLE_OBJECTIVE, + codexObjectiveAliases: [], + goals: [makeGoal()], + ...overrides, + }; +} + +function entry(kind: UlwLoopLedgerEntry["kind"], goalId = "G001"): UlwLoopLedgerEntry { + return { at: NOW, kind, goalId }; +} + +async function makeRepo(): Promise { + return mkdtemp(join(tmpdir(), "ug-io-")); +} + +async function writeRawPlan(repoRoot: string, plan: UlwLoopPlan): Promise { + await mkdir(ulwLoopDir(repoRoot), { recursive: true }); + await writeFile(ulwLoopGoalsPath(repoRoot), `${JSON.stringify(plan, null, 2)}\n`, "utf8"); +} + +async function readLedgerLines(repoRoot: string): Promise { + const raw = await readFile(ulwLoopLedgerPath(repoRoot), "utf8"); + return raw.split(/\r?\n/).filter(Boolean); +} + +describe("readUlwLoopPlan", () => { + let repoRoot = ""; + + beforeEach(async () => { + // given + repoRoot = await makeRepo(); + }); + + it("throws UlwLoopError when goals.json is missing", async () => { + // when/then + await expect(readUlwLoopPlan(repoRoot)).rejects.toThrow(UlwLoopError); + await expect(readUlwLoopPlan(repoRoot)).rejects.toThrow("omo ulw-loop create-goals"); + }); + + it("returns parsed plan when fixture is present", async () => { + // given + await mkdir(ulwLoopDir(repoRoot), { recursive: true }); + await copyFile(join(process.cwd(), "test", "fixtures", "sample-plan.json"), ulwLoopGoalsPath(repoRoot)); + + // when + const plan = await readUlwLoopPlan(repoRoot); + + // then + expect(plan.version).toBe(1); + expect(plan.codexGoalMode).toBe("aggregate"); + expect(plan.goals).toHaveLength(3); + expect(plan.goals[0]?.successCriteria).toHaveLength(3); + }); + + it("migrates legacy aggregate objective on read + writes aggregate_objective_migrated ledger entry + retains alias", async () => { + // given + const legacyObjective = "Complete all ulw-loop stories in .omo/ulw-loop/goals.json: G001 Build auth service"; + await writeRawPlan(repoRoot, makePlan({ codexObjective: legacyObjective })); + + // when + const plan = await readUlwLoopPlan(repoRoot); + + // then + expect(plan.codexObjective).toBe(STABLE_OBJECTIVE); + expect(plan.codexObjectiveAliases).toContain(legacyObjective); + const persisted = JSON.parse(await readFile(ulwLoopGoalsPath(repoRoot), "utf8")); + expect(persisted).toMatchObject({ codexObjective: STABLE_OBJECTIVE, codexObjectiveAliases: [legacyObjective] }); + const lines = await readLedgerLines(repoRoot); + expect(lines).toHaveLength(1); + expect(JSON.parse(lines[0] ?? "{}")).toMatchObject({ + kind: "aggregate_objective_migrated", + before: { codexObjective: legacyObjective }, + }); + }); +}); + +describe("writePlan", () => { + it("writes goals.json atomically with no temp file left behind", async () => { + // given + const repoRoot = await makeRepo(); + + // when + await writePlan(repoRoot, makePlan()); + + // then + const raw = await readFile(ulwLoopGoalsPath(repoRoot), "utf8"); + expect(JSON.parse(raw)).toMatchObject({ version: 1, goals: [{ id: "G001" }] }); + expect((await readdir(ulwLoopDir(repoRoot))).filter((name) => name.endsWith(".tmp"))).toEqual([]); + }); + + it("overwrites existing file", async () => { + // given + const repoRoot = await makeRepo(); + await writePlan(repoRoot, makePlan({ codexObjective: "first" })); + + // when + await writePlan(repoRoot, makePlan({ codexObjective: "second" })); + + // then + expect(JSON.parse(await readFile(ulwLoopGoalsPath(repoRoot), "utf8"))).toMatchObject({ + codexObjective: "second", + }); + }); +}); + +describe("appendLedger", () => { + it("appends a single JSONL line to ledger.jsonl", async () => { + // given + const repoRoot = await makeRepo(); + const ledgerEntry = entry("goal_started"); + + // when + await appendLedger(repoRoot, ledgerEntry); + + // then + expect(await readLedgerLines(repoRoot)).toEqual([JSON.stringify(ledgerEntry)]); + }); + + it("creates ledger.jsonl if missing", async () => { + // given + const repoRoot = await makeRepo(); + + // when + await appendLedger(repoRoot, entry("goal_completed")); + + // then + expect(await readFile(ulwLoopLedgerPath(repoRoot), "utf8")).toContain("goal_completed"); + }); + + it("preserves prior entries", async () => { + // given + const repoRoot = await makeRepo(); + const first = entry("goal_started"); + const second = entry("goal_completed"); + + // when + await appendLedger(repoRoot, first); + await appendLedger(repoRoot, second); + + // then + expect(await readLedgerLines(repoRoot)).toEqual([JSON.stringify(first), JSON.stringify(second)]); + }); +}); + +describe("readSteeringLedgerEntries", () => { + it("returns only steering-related event kinds", async () => { + // given + const repoRoot = await makeRepo(); + await appendLedger(repoRoot, entry("steering_accepted")); + await appendLedger(repoRoot, entry("goal_started")); + await appendLedger(repoRoot, entry("steering_rejected")); + await appendLedger(repoRoot, entry("criteria_revised")); + + // when + const entries = await readSteeringLedgerEntries(repoRoot); + + // then + expect(entries.map((item) => item.kind)).toEqual(["steering_accepted", "steering_rejected", "criteria_revised"]); + }); + + it("returns empty array when ledger missing", async () => { + // given + const repoRoot = await makeRepo(); + + // when/then + await expect(readSteeringLedgerEntries(repoRoot)).resolves.toEqual([]); + }); +}); + +describe("withUlwLoopMutationLock", () => { + it("serializes concurrent invocations", async () => { + // given + const repoRoot = await makeRepo(); + const counterPath = join(repoRoot, "counter.txt"); + let active = 0; + let maxActive = 0; + await writeFile(counterPath, "0", "utf8"); + + // when + await Promise.all( + [1, 2, 3].map((_) => + withUlwLoopMutationLock(repoRoot, async () => { + active += 1; + maxActive = Math.max(maxActive, active); + const current = Number(await readFile(counterPath, "utf8")); + await Promise.resolve(); + await writeFile(counterPath, String(current + 1), "utf8"); + active -= 1; + }), + ), + ); + + // then + expect(maxActive).toBe(1); + expect(await readFile(counterPath, "utf8")).toBe("3"); + }); +}); diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/quality-gate.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/quality-gate.test.ts new file mode 100644 index 000000000..9dd0aebc0 --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/quality-gate.test.ts @@ -0,0 +1,203 @@ +import { readFile } from "node:fs/promises"; + +import { describe, expect, it } from "vitest"; + +import { + classifyExternalAuthorizationBlocker, + clearGoalBlockerFields, + normalizeBlockerEvidence, + sameBlockerOccurrences, + validateQualityGate, +} from "../src/quality-gate.js"; +import type { UlwLoopItem, UlwLoopPlan } from "../src/types.js"; +import { UlwLoopError } from "../src/types.js"; + +const NOW = "2026-05-23T00:00:00.000Z"; +const VALID_GATE = { + aiSlopCleaner: { status: "passed", evidence: "no slop detected after cleaner run" }, + verification: { status: "passed", commands: ["npm test"], evidence: "all tests pass" }, + codeReview: { recommendation: "APPROVE", architectStatus: "CLEAR", evidence: "ship it" }, + criteriaCoverage: { totalCriteria: 2, passCount: 2, adversarialClassesCovered: ["malformed_input"] }, +} as const; + +interface GoalWithBlocker extends UlwLoopItem { + blocker?: { readonly signature: string }; + blockerEvidence?: string; + blockerOccurrences?: number; + blockedAt?: string; +} + +function makeGate(overrides: Record = {}): Record { + return { ...VALID_GATE, ...overrides }; +} + +function getQualityGateError(input: unknown): UlwLoopError { + try { + validateQualityGate(input); + } catch (error) { + if (error instanceof UlwLoopError) return error; + throw error; + } + throw new Error("Expected UlwLoopError"); +} + +function makeGoal(overrides: Partial = {}): UlwLoopItem { + return { + id: "G001", + title: "Goal one", + objective: "Complete goal one", + status: "pending", + successCriteria: [], + attempt: 1, + createdAt: NOW, + updatedAt: NOW, + ...overrides, + }; +} + +function makePlan(goals: UlwLoopItem[]): UlwLoopPlan { + return { + version: 1, + createdAt: NOW, + updatedAt: NOW, + briefPath: ".omo/ulw-loop/brief.md", + goalsPath: ".omo/ulw-loop/goals.json", + ledgerPath: ".omo/ulw-loop/ledger.jsonl", + goals, + }; +} + +describe("validateQualityGate", () => { + it("accepts valid quality gate from fixture", async () => { + // given + const raw = await readFile(new URL("./fixtures/sample-quality-gate.json", import.meta.url), "utf8"); + const parsed: unknown = JSON.parse(raw); + + // when + const gate = validateQualityGate(parsed); + + // then + expect(gate.aiSlopCleaner.status).toBe("passed"); + expect(gate).toMatchObject({ criteriaCoverage: { totalCriteria: 9, passCount: 9 } }); + }); + + it("throws UlwLoopError when aiSlopCleaner missing", () => { + // when + const error = getQualityGateError(makeGate({ aiSlopCleaner: undefined })); + + // then + expect(error.code).toBe("ULW_LOOP_QUALITY_GATE_INVALID"); + }); + + it("throws UlwLoopError when verification missing", () => { + // when + const error = getQualityGateError(makeGate({ verification: undefined })); + + // then + expect(error.code).toBe("ULW_LOOP_QUALITY_GATE_INVALID"); + }); + + it("throws UlwLoopError when codeReview missing", () => { + // when + const error = getQualityGateError(makeGate({ codeReview: undefined })); + + // then + expect(error.code).toBe("ULW_LOOP_QUALITY_GATE_INVALID"); + }); + + it("throws UlwLoopError when criteriaCoverage missing (NEW)", () => { + // when + const error = getQualityGateError(makeGate({ criteriaCoverage: undefined })); + + // then + expect(error.code).toBe("ULW_LOOP_QUALITY_GATE_INVALID"); + }); + + it("throws UlwLoopError when criteriaCoverage.passCount < totalCriteria (NEW)", () => { + // when + const error = getQualityGateError( + makeGate({ criteriaCoverage: { totalCriteria: 3, passCount: 2, adversarialClassesCovered: [] } }), + ); + + // then + expect(error.message).toContain("criteriaCoverage.passCount"); + }); + + it("throws UlwLoopError when codeReview.recommendation is not APPROVE", () => { + // when + const error = getQualityGateError( + makeGate({ codeReview: { ...VALID_GATE.codeReview, recommendation: "COMMENT" } }), + ); + + // then + expect(error.message).toContain("recommendation"); + }); + + it("throws UlwLoopError when architectStatus is not CLEAR", () => { + // when + const error = getQualityGateError( + makeGate({ codeReview: { ...VALID_GATE.codeReview, architectStatus: "WATCH" } }), + ); + + // then + expect(error.message).toContain("architectStatus"); + }); +}); + +describe("classifyExternalAuthorizationBlocker", () => { + it("returns GHCR signature when evidence mentions ghcr.io auth failure", () => { + expect( + classifyExternalAuthorizationBlocker("ghcr.io returned 401 authentication required for package pull"), + ).toBe("GHCR_PULL_ACCESS:HTTP_401_ANONYMOUS:GHCR_VISIBILITY_OR_CREDENTIAL_REQUIRED"); + }); + + it("returns generic auth signature for generic 401 evidence", () => { + expect(classifyExternalAuthorizationBlocker("Registry returned 401 because credentials are missing")).toBe( + "EXTERNAL_AUTHORIZATION_REQUIRED", + ); + }); + + it("returns null when no auth keywords", () => { + expect(classifyExternalAuthorizationBlocker("build failed because tests failed")).toBeNull(); + }); +}); + +describe("normalizeBlockerEvidence", () => { + it("collapses whitespace + lowercases", () => { + expect(normalizeBlockerEvidence(" GHCR.IO\n\tNeeds TOKEN ")).toBe("ghcr.io needs token"); + }); +}); + +describe("sameBlockerOccurrences", () => { + it("counts goals matching signature", () => { + // given + const nested: GoalWithBlocker = { ...makeGoal({ id: "G002" }), blocker: { signature: "AUTH" } }; + const plan = makePlan([makeGoal({ blockerSignature: "AUTH" }), nested, makeGoal({ id: "G003" })]); + + // when/then + expect(sameBlockerOccurrences(plan, "AUTH")).toBe(2); + }); +}); + +describe("clearGoalBlockerFields", () => { + it("clears all 5 blocker fields", () => { + // given + const goal: GoalWithBlocker = { + ...makeGoal({ blockerSignature: "AUTH" }), + blocker: { signature: "AUTH" }, + blockerEvidence: "401 unauthorized", + blockerOccurrences: 2, + blockedAt: NOW, + }; + + // when + clearGoalBlockerFields(goal); + + // then + expect(goal).not.toHaveProperty("blocker"); + expect(goal).not.toHaveProperty("blockerSignature"); + expect(goal).not.toHaveProperty("blockerEvidence"); + expect(goal).not.toHaveProperty("blockerOccurrences"); + expect(goal).not.toHaveProperty("blockedAt"); + }); +}); diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/review-blockers.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/review-blockers.test.ts new file mode 100644 index 000000000..025133713 --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/review-blockers.test.ts @@ -0,0 +1,180 @@ +import { mkdir, mkdtemp, readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE } from "../src/goal-status.js"; +import { ulwLoopDir, ulwLoopLedgerPath } from "../src/paths.js"; +import { writePlan } from "../src/plan-io.js"; +import { recordFinalReviewBlockers } from "../src/review-blockers.js"; +import type { UlwLoopItem, UlwLoopPlan, UlwLoopSuccessCriterion } from "../src/types.js"; +import { UlwLoopError } from "../src/types.js"; + +const NOW = "2026-05-23T00:00:00.000Z"; +const VALID_SNAPSHOT_JSON = JSON.stringify({ + goal: { objective: ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE, status: "active" }, +}); + +const validArgs = { + goalId: "G002", + title: "Resolve final code-review blockers", + objective: "Address the BLOCK findings from the architect", + evidence: "review verdict: REQUEST_CHANGES (3 issues)", + codexGoalJson: VALID_SNAPSHOT_JSON, +}; + +function makeCriterion(overrides: Partial = {}): UlwLoopSuccessCriterion { + return { + id: "C001", + scenario: "happy path", + userModel: "happy", + expectedEvidence: "observable proof", + capturedEvidence: null, + status: "pending", + ...overrides, + }; +} + +function makeGoal(overrides: Partial = {}): UlwLoopItem { + return { + id: "G001", + title: "Build durable plan", + objective: "Complete one ulw-loop story", + status: "pending", + successCriteria: [makeCriterion()], + attempt: 1, + createdAt: NOW, + updatedAt: NOW, + ...overrides, + }; +} + +function makePlan(overrides: Partial = {}): UlwLoopPlan { + return { + version: 1, + createdAt: NOW, + updatedAt: NOW, + briefPath: ".omo/ulw-loop/brief.md", + goalsPath: ".omo/ulw-loop/goals.json", + ledgerPath: ".omo/ulw-loop/ledger.jsonl", + codexGoalMode: "aggregate", + codexObjective: ULW_LOOP_AGGREGATE_CODEX_OBJECTIVE, + goals: [makeGoal({ status: "in_progress" })], + ...overrides, + }; +} + +async function bootstrapRepo(plan: UlwLoopPlan): Promise { + const repo = await mkdtemp(join(tmpdir(), "ug-review-blockers-")); + await mkdir(ulwLoopDir(repo), { recursive: true }); + await writePlan(repo, plan); + return repo; +} + +async function ledgerKinds(repo: string): Promise { + const raw = await readFile(ulwLoopLedgerPath(repo), "utf8"); + return raw + .split(/\r?\n/) + .filter(Boolean) + .map((line) => JSON.parse(line).kind); +} + +async function expectUlwLoopCode(action: () => Promise, code: string): Promise { + try { + await action(); + } catch (error) { + expect(error).toBeInstanceOf(UlwLoopError); + if (!(error instanceof UlwLoopError)) throw error; + expect(error.code).toBe(code); + return; + } + throw new Error("Expected UlwLoopError"); +} + +function finalPlan(): UlwLoopPlan { + return makePlan({ + activeGoalId: "G002", + goals: [ + makeGoal({ id: "G001", status: "complete" }), + makeGoal({ id: "G002", status: "in_progress", title: "ship it", objective: "Finish final story" }), + ], + }); +} + +describe("recordFinalReviewBlockers happy path", () => { + it("marks the final goal review_blocked + appends new pending goal", async () => { + const repo = await bootstrapRepo(finalPlan()); + + const result = await recordFinalReviewBlockers(repo, validArgs); + + expect(result.blockedGoal.status).toBe("review_blocked"); + expect(result.blockedGoal.evidence).toBe(validArgs.evidence); + expect(result.newGoal).toMatchObject({ id: "G003", status: "pending", title: validArgs.title }); + expect(result.newGoal.successCriteria.length).toBeGreaterThanOrEqual(3); + expect(result.plan.activeGoalId).toBeUndefined(); + expect(result.ledgerEntries.length).toBeGreaterThanOrEqual(3); + }); + + it("seeded successCriteria cover happy/edge/regression on the blocker-resolution goal", async () => { + const repo = await bootstrapRepo(finalPlan()); + + const result = await recordFinalReviewBlockers(repo, validArgs); + + expect(result.newGoal.successCriteria.map((criterion) => criterion.userModel).sort()).toEqual([ + "edge", + "happy", + "regression", + ]); + }); +}); + +describe("recordFinalReviewBlockers error cases", () => { + it("throws ulw_loop_goal_not_found for unknown goalId", async () => { + const repo = await bootstrapRepo(finalPlan()); + await expectUlwLoopCode( + () => recordFinalReviewBlockers(repo, { ...validArgs, goalId: "G999" }), + "ulw_loop_goal_not_found", + ); + }); + + it("throws ulw_loop_goal_not_in_progress when goal.status !== in_progress", async () => { + const repo = await bootstrapRepo( + makePlan({ + goals: [makeGoal({ id: "G001", status: "in_progress" }), makeGoal({ id: "G002", status: "pending" })], + }), + ); + await expectUlwLoopCode(() => recordFinalReviewBlockers(repo, validArgs), "ulw_loop_goal_not_in_progress"); + }); + + it("throws ulw_loop_not_final_story when other unresolved goals remain", async () => { + const repo = await bootstrapRepo( + makePlan({ + goals: [makeGoal({ id: "G001", status: "in_progress" }), makeGoal({ id: "G002", status: "pending" })], + }), + ); + await expectUlwLoopCode( + () => recordFinalReviewBlockers(repo, { ...validArgs, goalId: "G001" }), + "ulw_loop_not_final_story", + ); + }); + + it("throws ulw_loop_codex_snapshot_mismatch when objective mismatches", async () => { + const repo = await bootstrapRepo(finalPlan()); + const codexGoalJson = JSON.stringify({ goal: { objective: "wrong", status: "active" } }); + + await expectUlwLoopCode( + () => recordFinalReviewBlockers(repo, { ...validArgs, codexGoalJson }), + "ulw_loop_codex_snapshot_mismatch", + ); + }); +}); + +describe("recordFinalReviewBlockers ledger entries", () => { + it("appends goal_review_blocked + goal_added + blocker_recorded events", async () => { + const repo = await bootstrapRepo(finalPlan()); + + await recordFinalReviewBlockers(repo, validArgs); + + expect(await ledgerKinds(repo)).toEqual(["goal_review_blocked", "goal_added", "blocker_recorded"]); + }); +}); diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/steering.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/steering.test.ts new file mode 100644 index 000000000..674bc14f5 --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/steering.test.ts @@ -0,0 +1,353 @@ +import { mkdtemp, readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { ulwLoopGoalsPath } from "../src/paths.js"; +import { readSteeringLedgerEntries, readUlwLoopPlan, writePlan } from "../src/plan-io.js"; +import { + applySteeringMutation, + parseUlwLoopSteeringDirective, + steerUlwLoop, + validateUlwLoopSteeringProposal, +} from "../src/steering.js"; +import type { + UlwLoopItem, + UlwLoopPlan, + UlwLoopSteeringProposal, + UlwLoopSuccessCriterion, + UlwLoopSuccessCriterionUserModel, +} from "../src/types.js"; + +const NOW = "2026-05-23T00:00:00.000Z"; + +type CriterionSteeringFields = { + readonly goalId?: string; + readonly scenario?: string; + readonly expectedEvidence?: string; + readonly userModel?: UlwLoopSuccessCriterionUserModel; +}; +type SteeringInput = UlwLoopSteeringProposal & CriterionSteeringFields; + +function criterion(overrides: Partial = {}): UlwLoopSuccessCriterion { + return { + id: "C001", + scenario: "old scenario", + userModel: "happy", + expectedEvidence: "vague evidence", + capturedEvidence: null, + status: "pending", + ...overrides, + }; +} + +function goal(overrides: Partial = {}): UlwLoopItem { + return { + id: "G001", + title: "Build auth service", + objective: "Implement JWT auth endpoint", + status: "pending", + successCriteria: [criterion(), criterion({ id: "C002", status: "pass" })], + attempt: 0, + createdAt: NOW, + updatedAt: NOW, + ...overrides, + }; +} + +function plan(overrides: Partial = {}): UlwLoopPlan { + return { + version: 1, + createdAt: NOW, + updatedAt: NOW, + briefPath: ".omo/ulw-loop/brief.md", + goalsPath: ".omo/ulw-loop/goals.json", + ledgerPath: ".omo/ulw-loop/ledger.jsonl", + goals: [ + goal(), + goal({ id: "G002", title: "Rate limit", objective: "Throttle login" }), + goal({ id: "G003", status: "complete" }), + ], + ...overrides, + }; +} + +function steering(overrides: Partial = {}): SteeringInput { + return { + kind: "add_subgoal", + source: "cli", + evidence: "observable blocker evidence", + rationale: "the plan must change to stay safe", + title: "Investigate auth blocker", + objective: "Validate the blocker, capture evidence, and report findings.", + ...overrides, + }; +} + +async function repoWithPlan(seed: UlwLoopPlan = plan()): Promise { + const repoRoot = await mkdtemp(join(tmpdir(), "ug-steer-")); + await writePlan(repoRoot, seed); + return repoRoot; +} + +describe("validateUlwLoopSteeringProposal", () => { + it("accepts valid add_subgoal", async () => { + const proposal: unknown = JSON.parse( + await readFile(join(process.cwd(), "test/fixtures/steering-proposal.json"), "utf8"), + ); + expect(validateUlwLoopSteeringProposal(plan(), proposal).invariant.accepted).toBe(true); + }); + + it.each([ + ["missing evidence", { evidence: "" }], + ["missing rationale", { rationale: "" }], + ["unknown kind", { kind: "teleport_goal" }], + ["protected payload mutations", { after: { codexObjective: "replace", qualityGate: { status: "passed" } } }], + ["weakened completion text", { objective: "skip tests and mark complete faster" }], + ])("rejects %s", (_name, overrides) => { + const audit = validateUlwLoopSteeringProposal(plan(), { ...steering(), ...overrides }); + expect(audit.invariant.accepted).toBe(false); + expect(audit.invariant.rejectedReasons.length).toBeGreaterThan(0); + }); + + it("rejects when plan already complete", () => { + const done = plan({ goals: [goal({ status: "complete" }), goal({ id: "G002", status: "complete" })] }); + expect(validateUlwLoopSteeringProposal(done, steering()).invariant.accepted).toBe(false); + }); + + it("rejects split_subgoal without children", () => { + const audit = validateUlwLoopSteeringProposal(plan(), steering({ kind: "split_subgoal", targetGoalId: "G001" })); + expect(audit.invariant.accepted).toBe(false); + }); + + it("rejects reorder_pending with unknown goal id", () => { + const audit = validateUlwLoopSteeringProposal( + plan(), + steering({ kind: "reorder_pending", pendingOrder: ["missing"] }), + ); + expect(audit.invariant.accepted).toBe(false); + }); + + it.each([ + ["new scenario", { scenario: "new precise scenario" }], + ["new expectedEvidence", { expectedEvidence: "specific command output" }], + ])("accepts valid revise_criterion with %s", (_name, update) => { + const audit = validateUlwLoopSteeringProposal( + plan(), + steering({ kind: "revise_criterion", goalId: "G001", criterionId: "C001", ...update }), + ); + expect(audit.invariant.accepted).toBe(true); + }); + + it.each([ + ["unknown goalId", { goalId: "missing", criterionId: "C001", scenario: "new" }], + ["unknown criterionId", { goalId: "G001", criterionId: "missing", scenario: "new" }], + ["no updates", { goalId: "G001", criterionId: "C001" }], + ])("rejects revise_criterion with %s", (_name, overrides) => { + const audit = validateUlwLoopSteeringProposal(plan(), steering({ kind: "revise_criterion", ...overrides })); + expect(audit.invariant.accepted).toBe(false); + }); +}); + +describe("steerUlwLoop", () => { + describe("steering-created goals", () => { + function sluggedPlan(): UlwLoopPlan { + return plan({ + goals: [ + goal({ id: "G001-goal-a", title: "Goal A", objective: "Do A" }), + goal({ id: "G002-goal-b", title: "Goal B", objective: "Do B" }), + ], + }); + } + + it("add_subgoal: uses next numeric id + default success criteria", async () => { + const repoRoot = await repoWithPlan(sluggedPlan()); + const result = await steerUlwLoop(repoRoot, steering({ idempotencyKey: "slug-add" })); + expect(result.plan.goals.at(-1)).toMatchObject({ + id: "G003", + successCriteria: [{ id: "C001" }, { id: "C002" }, { id: "C003" }], + }); + }); + + it("split_subgoal: replacement goals use default success criteria", async () => { + const repoRoot = await repoWithPlan(sluggedPlan()); + const result = await steerUlwLoop( + repoRoot, + steering({ + kind: "split_subgoal", + targetGoalId: "G001-goal-a", + childGoals: [{ title: "Child A", objective: "Do child A" }], + }), + ); + expect(result.plan.goals[1]).toMatchObject({ + id: "G003", + successCriteria: [{ id: "C001" }, { id: "C002" }, { id: "C003" }], + }); + }); + + it("mark_blocked_superseded: replacement goals use default success criteria", async () => { + const repoRoot = await repoWithPlan(sluggedPlan()); + const result = await steerUlwLoop( + repoRoot, + steering({ + kind: "mark_blocked_superseded", + targetGoalId: "G001-goal-a", + childGoals: [{ title: "Replacement", objective: "Replace blocked path" }], + }), + ); + expect(result.plan.goals[1]).toMatchObject({ + id: "G003", + successCriteria: [{ id: "C001" }, { id: "C002" }, { id: "C003" }], + }); + }); + }); + + it("add_subgoal: appends goal + ledger entry", async () => { + const repoRoot = await repoWithPlan(); + const result = await steerUlwLoop(repoRoot, steering({ idempotencyKey: "add" })); + const persisted = await readUlwLoopPlan(repoRoot); + expect(result.accepted).toBe(true); + expect(persisted.goals.at(-1)).toMatchObject({ id: "G004", title: "Investigate auth blocker" }); + expect((await readSteeringLedgerEntries(repoRoot)).at(-1)).toMatchObject({ + kind: "steering_accepted", + mutationKind: "add_subgoal", + }); + }); + + it("split_subgoal: creates children + supersedes parent", async () => { + const repoRoot = await repoWithPlan(); + const result = await steerUlwLoop( + repoRoot, + steering({ + kind: "split_subgoal", + targetGoalId: "G001", + childGoals: [{ title: "Child", objective: "Do child" }], + }), + ); + expect(result.plan.goals.map((item) => item.id).slice(0, 2)).toEqual(["G001", "G004"]); + expect(result.plan.goals[0]).toMatchObject({ steeringStatus: "superseded", supersededBy: ["G004"] }); + }); + + it("reorder_pending: changes goal order", async () => { + const repoRoot = await repoWithPlan(); + const result = await steerUlwLoop( + repoRoot, + steering({ kind: "reorder_pending", pendingOrder: ["G002", "G001"] }), + ); + expect(result.plan.goals.map((item) => item.id).slice(0, 2)).toEqual(["G002", "G001"]); + }); + + it("revise_pending_wording: updates title/objective", async () => { + const repoRoot = await repoWithPlan(); + const result = await steerUlwLoop( + repoRoot, + steering({ + kind: "revise_pending_wording", + targetGoalId: "G001", + revisedTitle: "Build safer auth", + revisedObjective: "Implement guarded JWT auth", + }), + ); + expect(result.plan.goals[0]).toMatchObject({ + title: "Build safer auth", + objective: "Implement guarded JWT auth", + }); + }); + + it("annotate_ledger: ledger-only, no plan mutation", async () => { + const seed = plan(); + const repoRoot = await repoWithPlan(seed); + const result = await steerUlwLoop(repoRoot, steering({ kind: "annotate_ledger" })); + expect(result.plan.goals).toEqual(seed.goals); + expect(await readFile(ulwLoopGoalsPath(repoRoot), "utf8")).toBe(`${JSON.stringify(seed, null, 2)}\n`); + }); + + it("mark_blocked_superseded with children: supersede + replace", async () => { + const repoRoot = await repoWithPlan(); + const result = await steerUlwLoop( + repoRoot, + steering({ + kind: "mark_blocked_superseded", + targetGoalId: "G001", + childGoals: [{ title: "Replacement", objective: "Replace blocked path" }], + }), + ); + expect(result.plan.goals[0]).toMatchObject({ steeringStatus: "superseded", supersededBy: ["G004"] }); + expect(result.plan.goals[1]).toMatchObject({ id: "G004", supersedes: ["G001"] }); + }); + + it("mark_blocked_superseded without children: blocks goal", async () => { + const repoRoot = await repoWithPlan(); + const result = await steerUlwLoop( + repoRoot, + steering({ kind: "mark_blocked_superseded", targetGoalId: "G001", blockedReason: "external blocker" }), + ); + expect(result.plan.goals[0]).toMatchObject({ + status: "blocked", + steeringStatus: "blocked", + blockedReason: "external blocker", + }); + }); + + it.each(["pending", "pass"] as const)("revise_criterion: works on a %s criterion", async (status) => { + const repoRoot = await repoWithPlan(); + const criterionId = status === "pending" ? "C001" : "C002"; + const result = await steerUlwLoop( + repoRoot, + steering({ + kind: "revise_criterion", + goalId: "G001", + criterionId, + scenario: "new scenario", + expectedEvidence: "precise evidence", + }), + ); + const updated = result.plan.goals[0]?.successCriteria.find((item) => item.id === criterionId); + expect(updated).toMatchObject({ scenario: "new scenario", expectedEvidence: "precise evidence", status }); + expect((await readSteeringLedgerEntries(repoRoot)).at(-1)).toMatchObject({ + kind: "criteria_revised", + criterionId, + }); + }); + + it("revise_criterion: updates the targeted criterion in plan", () => { + const audit = validateUlwLoopSteeringProposal( + plan(), + steering({ kind: "revise_criterion", goalId: "G001", criterionId: "C001", scenario: "new value" }), + ); + const next = applySteeringMutation( + plan(), + steering({ kind: "revise_criterion", goalId: "G001", criterionId: "C001", scenario: "new value" }), + audit, + ); + expect(next.goals[0]?.successCriteria[0]?.scenario).toBe("new value"); + }); + + it("idempotency: same idempotencyKey produces deduped true second time", async () => { + const repoRoot = await repoWithPlan(); + await steerUlwLoop(repoRoot, steering({ idempotencyKey: "same-key" })); + const second = await steerUlwLoop(repoRoot, steering({ idempotencyKey: "same-key" })); + expect(second.deduped).toBe(true); + expect((await readUlwLoopPlan(repoRoot)).goals).toHaveLength(4); + }); +}); + +describe("parseUlwLoopSteeringDirective", () => { + it.each(["OMO_ULW_LOOP_STEER", "omo.ulw-loop.steer", "omo ulw-loop steer"])("parses %s pattern", (marker) => { + expect(parseUlwLoopSteeringDirective(`${marker}: ${JSON.stringify(steering())}`)).toMatchObject({ + kind: "add_subgoal", + }); + }); + + it("returns null when no marker", () => { + expect(parseUlwLoopSteeringDirective(JSON.stringify(steering()))).toBeNull(); + }); + + it("returns null when JSON malformed after marker", () => { + expect(parseUlwLoopSteeringDirective("OMO_ULW_LOOP_STEER: {bad json")).toBeNull(); + }); + + it("returns null for deprecated markers", () => { + const marker = ["OM", "X_ULW_LOOP_STEER"].join(""); + expect(parseUlwLoopSteeringDirective(`${marker}: ${JSON.stringify(steering())}`)).toBeNull(); + }); +}); diff --git a/packages/omo-codex/plugin/components/ulw-loop/test/types.test.ts b/packages/omo-codex/plugin/components/ulw-loop/test/types.test.ts new file mode 100644 index 000000000..794f17ee6 --- /dev/null +++ b/packages/omo-codex/plugin/components/ulw-loop/test/types.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "vitest"; + +import { + iso, + ULW_LOOP_BRIEF, + ULW_LOOP_CRITERION_STATUSES, + ULW_LOOP_DIR, + ULW_LOOP_GOALS, + ULW_LOOP_LEDGER, + ULW_LOOP_STEERING_MUTATION_KINDS, + ULW_LOOP_SUCCESS_CRITERION_USER_MODELS, + UlwLoopError, +} from "../src/types.ts"; + +describe("ulw-loop domain constants", () => { + describe("when checking workspace paths", () => { + it("then ULW_LOOP_DIR points to the omo workspace", () => { + expect(ULW_LOOP_DIR).toBe(".omo/ulw-loop"); + }); + + it("then artifact filenames are stable", () => { + expect(ULW_LOOP_BRIEF).toBe("brief.md"); + expect(ULW_LOOP_GOALS).toBe("goals.json"); + expect(ULW_LOOP_LEDGER).toBe("ledger.jsonl"); + }); + }); + + describe("when checking steering mutation kinds", () => { + it("then includes the new revise_criterion kind", () => { + expect(ULW_LOOP_STEERING_MUTATION_KINDS).toContain("revise_criterion"); + }); + + it("then totals 7 kinds", () => { + expect(ULW_LOOP_STEERING_MUTATION_KINDS).toHaveLength(7); + }); + }); + + describe("when checking criterion user models", () => { + it("then exposes 4 user models including adversarial", () => { + expect(ULW_LOOP_SUCCESS_CRITERION_USER_MODELS).toEqual(["happy", "edge", "regression", "adversarial"]); + }); + }); + + describe("when checking criterion statuses", () => { + it("then exposes pending/pass/fail/blocked", () => { + expect(ULW_LOOP_CRITERION_STATUSES).toEqual(["pending", "pass", "fail", "blocked"]); + }); + }); +}); + +describe("UlwLoopError", () => { + describe("when constructed with code", () => { + it("then is an Error instance carrying the code", () => { + const err = new UlwLoopError("bad", "TEST_CODE"); + + expect(err).toBeInstanceOf(Error); + expect(err.code).toBe("TEST_CODE"); + expect(err.message).toBe("bad"); + }); + + it("then accepts optional cause + details", () => { + const cause = new Error("upstream"); + const err = new UlwLoopError("wrap", "WRAP", { cause, details: { goalId: "G001" } }); + + expect(err.cause).toBe(cause); + expect(err.details).toEqual({ goalId: "G001" }); + }); + }); +}); + +describe("iso()", () => { + describe("when called", () => { + it("then returns an ISO 8601 string", () => { + const s = iso(); + + expect(s).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/); + }); + }); +});