From da446979f6f96a9b1f54beda96a7806fb6b4000a Mon Sep 17 00:00:00 2001 From: ismeth Date: Fri, 27 Feb 2026 16:46:38 +0100 Subject: [PATCH] test(athena): add tests for intent addendums, hook enforcement, restriction parity, and council prompt regression --- .../athena/council-member-agent.test.ts | 22 +++ src/hooks/athena-sisyphus-only/index.test.ts | 104 ++++++++++++ .../agent-tool-restrictions-parity.test.ts | 2 +- src/shared/agent-tool-restrictions.test.ts | 2 - .../prepare-council-prompt/tools.test.ts | 152 ++++++++++++++++++ 5 files changed, 279 insertions(+), 3 deletions(-) create mode 100644 src/hooks/athena-sisyphus-only/index.test.ts create mode 100644 src/tools/prepare-council-prompt/tools.test.ts diff --git a/src/agents/athena/council-member-agent.test.ts b/src/agents/athena/council-member-agent.test.ts index a541ac6f1..82fe1a2c1 100644 --- a/src/agents/athena/council-member-agent.test.ts +++ b/src/agents/athena/council-member-agent.test.ts @@ -20,5 +20,27 @@ describe("COUNCIL_MEMBER_PROMPT", () => { expect(COUNCIL_MEMBER_PROMPT).toContain("") }) }) + + describe("#when checking for audit bias regression", () => { + it("#then does not contain severity (moved to AUDIT addendum)", () => { + expect(COUNCIL_MEMBER_PROMPT).not.toContain("severity") + }) + + it("#then does not contain Search the codebase (codebase-specific)", () => { + expect(COUNCIL_MEMBER_PROMPT).not.toContain("Search the codebase") + }) + + it("#then does not contain Focus on finding real issues (AUDIT-specific)", () => { + expect(COUNCIL_MEMBER_PROMPT).not.toContain("Focus on finding real issues") + }) + + it("#then does not contain AUDIT-style numbered finding headers", () => { + expect(COUNCIL_MEMBER_PROMPT).not.toMatch(/## Finding \d/) + }) + + it("#then contains evidence-based (generic analysis language)", () => { + expect(COUNCIL_MEMBER_PROMPT).toContain("evidence-based") + }) + }) }) }) diff --git a/src/hooks/athena-sisyphus-only/index.test.ts b/src/hooks/athena-sisyphus-only/index.test.ts new file mode 100644 index 000000000..056089497 --- /dev/null +++ b/src/hooks/athena-sisyphus-only/index.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from "bun:test" +import { isAllowedPath } from "./path-policy" +import { isAthenaAgent } from "./agent-matcher" + +const WORKSPACE_ROOT = "/fake/workspace" + +describe("athena-sisyphus-only hook", () => { + describe("#given the path policy", () => { + describe("#when checking allowed paths", () => { + it("#then allows .sisyphus/file.md", () => { + expect(isAllowedPath(".sisyphus/file.md", WORKSPACE_ROOT)).toBe(true) + }) + + it("#then allows nested .sisyphus/sub/dir/file.yaml", () => { + expect(isAllowedPath(".sisyphus/sub/dir/file.yaml", WORKSPACE_ROOT)).toBe(true) + }) + + it("#then allows any extension inside .sisyphus/", () => { + expect(isAllowedPath(".sisyphus/file.json", WORKSPACE_ROOT)).toBe(true) + }) + + it("#then allows deep nesting .sisyphus/notepads/plan/learnings.md", () => { + expect(isAllowedPath(".sisyphus/notepads/plan/learnings.md", WORKSPACE_ROOT)).toBe(true) + }) + }) + + describe("#when checking blocked paths", () => { + it("#then blocks src/agents/athena/agent.ts (outside .sisyphus/)", () => { + expect(isAllowedPath("src/agents/athena/agent.ts", WORKSPACE_ROOT)).toBe(false) + }) + + it("#then blocks docs/planning/synthesis.md (outside .sisyphus/)", () => { + expect(isAllowedPath("docs/planning/synthesis.md", WORKSPACE_ROOT)).toBe(false) + }) + + it("#then blocks root-level package.json", () => { + expect(isAllowedPath("package.json", WORKSPACE_ROOT)).toBe(false) + }) + + it("#then blocks absolute path outside project", () => { + expect(isAllowedPath("/etc/passwd", WORKSPACE_ROOT)).toBe(false) + }) + }) + + describe("#when checking path traversal attacks", () => { + it("#then blocks .sisyphus/../package.json (single traversal)", () => { + expect(isAllowedPath(".sisyphus/../package.json", WORKSPACE_ROOT)).toBe(false) + }) + + it("#then blocks .sisyphus/../../etc/passwd (double traversal)", () => { + expect(isAllowedPath(".sisyphus/../../etc/passwd", WORKSPACE_ROOT)).toBe(false) + }) + }) + + describe("#when checking edge cases", () => { + it("#then blocks empty string path", () => { + expect(isAllowedPath("", WORKSPACE_ROOT)).toBe(false) + }) + + it("#then blocks file named sisyphus without dot prefix", () => { + expect(isAllowedPath("sisyphus/file.md", WORKSPACE_ROOT)).toBe(false) + }) + + it("#then allows absolute path within .sisyphus/", () => { + const absPath = `${WORKSPACE_ROOT}/.sisyphus/plans/test.md` + expect(isAllowedPath(absPath, WORKSPACE_ROOT)).toBe(true) + }) + }) + }) + + describe("#given the agent matcher", () => { + describe("#when checking Athena agent", () => { + it("#then matches exact 'athena'", () => { + expect(isAthenaAgent("athena")).toBe(true) + }) + + it("#then matches case-insensitive 'Athena'", () => { + expect(isAthenaAgent("Athena")).toBe(true) + }) + + it("#then matches 'ATHENA' (all caps)", () => { + expect(isAthenaAgent("ATHENA")).toBe(true) + }) + }) + + describe("#when checking non-Athena agents", () => { + it("#then rejects undefined", () => { + expect(isAthenaAgent(undefined)).toBe(false) + }) + + it("#then rejects 'sisyphus'", () => { + expect(isAthenaAgent("sisyphus")).toBe(false) + }) + + it("#then rejects 'council-member'", () => { + expect(isAthenaAgent("council-member")).toBe(false) + }) + + it("#then rejects empty string", () => { + expect(isAthenaAgent("")).toBe(false) + }) + }) + }) +}) diff --git a/src/shared/agent-tool-restrictions-parity.test.ts b/src/shared/agent-tool-restrictions-parity.test.ts index 5dfe2e968..7f6f72309 100644 --- a/src/shared/agent-tool-restrictions-parity.test.ts +++ b/src/shared/agent-tool-restrictions-parity.test.ts @@ -14,7 +14,7 @@ import { getAgentToolRestrictions } from "./agent-tool-restrictions" // Surface 1: Athena deny-list from src/agents/athena/agent.ts // createAgentToolRestrictions(["write", "edit", "call_omo_agent"]) -const ATHENA_DENY_LIST = ["write", "edit", "call_omo_agent"] +const ATHENA_DENY_LIST = ["call_omo_agent"] // Surface 3: Council-member allowlist from src/agents/athena/council-member-agent.ts // createAgentToolAllowlist([...]) diff --git a/src/shared/agent-tool-restrictions.test.ts b/src/shared/agent-tool-restrictions.test.ts index 74c7fe1b5..0fe92fede 100644 --- a/src/shared/agent-tool-restrictions.test.ts +++ b/src/shared/agent-tool-restrictions.test.ts @@ -7,8 +7,6 @@ describe("agent-tool-restrictions", () => { //#when const restrictions = getAgentToolRestrictions("athena") //#then - expect(restrictions.write).toBe(false) - expect(restrictions.edit).toBe(false) expect(restrictions.call_omo_agent).toBe(false) }) diff --git a/src/tools/prepare-council-prompt/tools.test.ts b/src/tools/prepare-council-prompt/tools.test.ts new file mode 100644 index 000000000..3e5f07ad2 --- /dev/null +++ b/src/tools/prepare-council-prompt/tools.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it, afterEach } from "bun:test" +import { createPrepareCouncilPromptTool } from "./tools" +import { readFile, rm, mkdtemp } from "node:fs/promises" +import { join } from "node:path" +import { tmpdir } from "node:os" + +const mockContext = { + sessionID: "test-session", + messageID: "test-message", + agent: "test-agent", + abort: new AbortController().signal, +} + +function extractFilePath(result: string): string { + const match = result.match(/Council prompt saved to: (.+?) \(/) + if (!match) throw new Error(`Could not extract file path from result: ${result}`) + return match[1] +} + +describe("createPrepareCouncilPromptTool", () => { + let tmpDir: string + + afterEach(async () => { + if (tmpDir) { + await rm(tmpDir, { recursive: true, force: true }) + } + }) + + describe("#given a tool created with a temp directory", () => { + describe("#when called with intent AUDIT", () => { + it("#then produces file containing AUDIT addendum", async () => { + tmpDir = await mkdtemp(join(tmpdir(), "council-test-")) + const toolDef = createPrepareCouncilPromptTool(tmpDir) + const result = await toolDef.execute({ prompt: "Analyze the auth module", intent: "AUDIT" }, mockContext) + + const filePath = extractFilePath(result) + const content = await readFile(filePath, "utf-8") + expect(content).toContain("## Analysis Intent: AUDIT") + }) + }) + + describe("#when called with intent EVALUATE", () => { + it("#then produces file containing EVALUATE addendum", async () => { + tmpDir = await mkdtemp(join(tmpdir(), "council-test-")) + const toolDef = createPrepareCouncilPromptTool(tmpDir) + const result = await toolDef.execute({ prompt: "Compare REST vs GraphQL", intent: "EVALUATE" }, mockContext) + + const filePath = extractFilePath(result) + const content = await readFile(filePath, "utf-8") + expect(content).toContain("## Analysis Intent: EVALUATE") + }) + }) + + describe("#when called with intent PLAN", () => { + it("#then produces file containing PLAN addendum", async () => { + tmpDir = await mkdtemp(join(tmpdir(), "council-test-")) + const toolDef = createPrepareCouncilPromptTool(tmpDir) + const result = await toolDef.execute({ prompt: "Plan the migration to v2", intent: "PLAN" }, mockContext) + + const filePath = extractFilePath(result) + const content = await readFile(filePath, "utf-8") + expect(content).toContain("## Analysis Intent: PLAN") + }) + }) + + describe("#when called with intent EXPLAIN", () => { + it("#then produces file containing EXPLAIN addendum", async () => { + tmpDir = await mkdtemp(join(tmpdir(), "council-test-")) + const toolDef = createPrepareCouncilPromptTool(tmpDir) + const result = await toolDef.execute({ prompt: "How does the event loop work?", intent: "EXPLAIN" }, mockContext) + + const filePath = extractFilePath(result) + const content = await readFile(filePath, "utf-8") + expect(content).toContain("## Analysis Intent: EXPLAIN") + }) + }) + + describe("#when called without intent", () => { + it("#then defaults to AUDIT addendum", async () => { + tmpDir = await mkdtemp(join(tmpdir(), "council-test-")) + const toolDef = createPrepareCouncilPromptTool(tmpDir) + const result = await toolDef.execute({ prompt: "Review this module" }, mockContext) + + expect(result).toContain("intent: AUDIT") + const filePath = extractFilePath(result) + const content = await readFile(filePath, "utf-8") + expect(content).toContain("## Analysis Intent: AUDIT") + }) + }) + + describe("#when called with an invalid intent", () => { + it("#then returns an error message", async () => { + tmpDir = await mkdtemp(join(tmpdir(), "council-test-")) + const toolDef = createPrepareCouncilPromptTool(tmpDir) + const result = await toolDef.execute({ prompt: "Compare options", intent: "COMPARISON" }, mockContext) + + expect(result).toContain("Invalid intent") + expect(result).toContain("COMPARISON") + }) + }) + + describe("#when called with mode delegation and intent EVALUATE", () => { + it("#then file contains both delegation and EVALUATE addendums", async () => { + tmpDir = await mkdtemp(join(tmpdir(), "council-test-")) + const toolDef = createPrepareCouncilPromptTool(tmpDir) + const result = await toolDef.execute( + { prompt: "Evaluate caching strategies", mode: "delegation", intent: "EVALUATE" }, + mockContext, + ) + + const filePath = extractFilePath(result) + const content = await readFile(filePath, "utf-8") + expect(content).toContain("## Delegation Mode") + expect(content).toContain("## Analysis Intent: EVALUATE") + }) + }) + + describe("#when called with an empty prompt", () => { + it("#then returns an error about empty prompt", async () => { + tmpDir = await mkdtemp(join(tmpdir(), "council-test-")) + const toolDef = createPrepareCouncilPromptTool(tmpDir) + const result = await toolDef.execute({ prompt: "" }, mockContext) + + expect(result.toLowerCase()).toContain("empty") + }) + }) + + describe("#when checking file content order", () => { + it("#then mode addendum appears before intent addendum which appears before the question", async () => { + tmpDir = await mkdtemp(join(tmpdir(), "council-test-")) + const toolDef = createPrepareCouncilPromptTool(tmpDir) + const result = await toolDef.execute( + { prompt: "What is the deployment pipeline?", mode: "solo", intent: "PLAN" }, + mockContext, + ) + + const filePath = extractFilePath(result) + const content = await readFile(filePath, "utf-8") + + const modeIdx = content.indexOf("## Solo Analysis Mode") + const intentIdx = content.indexOf("## Analysis Intent: PLAN") + const questionIdx = content.indexOf("## Analysis Question") + + expect(modeIdx).toBeGreaterThanOrEqual(0) + expect(intentIdx).toBeGreaterThanOrEqual(0) + expect(questionIdx).toBeGreaterThanOrEqual(0) + expect(modeIdx).toBeLessThan(intentIdx) + expect(intentIdx).toBeLessThan(questionIdx) + }) + }) + }) +})