fix(athena): harden council flow and resilience config
This commit is contained in:
@@ -34,7 +34,6 @@ Agent factories following `createXXXAgent(model) → AgentConfig` pattern. Each
|
|||||||
| Multimodal-Looker | ALL except read |
|
| Multimodal-Looker | ALL except read |
|
||||||
| Atlas | task, call_omo_agent |
|
| Atlas | task, call_omo_agent |
|
||||||
| Momus | write, edit, task |
|
| Momus | write, edit, task |
|
||||||
| Athena | write, edit, call_omo_agent |
|
|
||||||
| Council-Member | ALL except read, grep, glob, lsp_*, ast_grep_search (allow-list) |
|
| Council-Member | ALL except read, grep, glob, lsp_*, ast_grep_search (allow-list) |
|
||||||
|
|
||||||
## STRUCTURE
|
## STRUCTURE
|
||||||
|
|||||||
@@ -14,6 +14,10 @@ describe("Athena prompt config injection placeholders", () => {
|
|||||||
expect(athenaConfig.prompt).toContain("{STUCK_THRESHOLD_SECONDS}")
|
expect(athenaConfig.prompt).toContain("{STUCK_THRESHOLD_SECONDS}")
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it("#then contains MEMBER_MAX_RUNNING_SECONDS placeholder", () => {
|
||||||
|
expect(athenaConfig.prompt).toContain("{MEMBER_MAX_RUNNING_SECONDS}")
|
||||||
|
})
|
||||||
|
|
||||||
it("#then contains quorum reference", () => {
|
it("#then contains quorum reference", () => {
|
||||||
expect(athenaConfig.prompt).toContain("quorum")
|
expect(athenaConfig.prompt).toContain("quorum")
|
||||||
})
|
})
|
||||||
@@ -29,6 +33,75 @@ describe("Athena prompt config injection placeholders", () => {
|
|||||||
it("#then contains CANCEL_RETRYING_ON_QUORUM placeholder", () => {
|
it("#then contains CANCEL_RETRYING_ON_QUORUM placeholder", () => {
|
||||||
expect(athenaConfig.prompt).toContain("{CANCEL_RETRYING_ON_QUORUM}")
|
expect(athenaConfig.prompt).toContain("{CANCEL_RETRYING_ON_QUORUM}")
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it("#then avoids session_id continuation retry instruction", () => {
|
||||||
|
expect(athenaConfig.prompt).not.toContain("Retries: use task(session_id=<member_session_id>")
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#then uses sequential workflow step numbering", () => {
|
||||||
|
expect(athenaConfig.prompt).toContain("Step 12: Synthesize")
|
||||||
|
expect(athenaConfig.prompt).toContain("Step 13: Determine the follow-up path")
|
||||||
|
expect(athenaConfig.prompt).toContain("Step 14: ACTIONABLE findings")
|
||||||
|
expect(athenaConfig.prompt).toContain("Step 15: INFORMATIONAL findings")
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#then omits legacy mixed step labels", () => {
|
||||||
|
expect(athenaConfig.prompt).not.toContain("Step 1.5")
|
||||||
|
expect(athenaConfig.prompt).not.toContain("Step 4.5")
|
||||||
|
expect(athenaConfig.prompt).not.toContain("Step 7A")
|
||||||
|
expect(athenaConfig.prompt).not.toContain("Step 7B")
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#then uses tagged prompt sections for consistency", () => {
|
||||||
|
expect(athenaConfig.prompt).toContain("<identity>")
|
||||||
|
expect(athenaConfig.prompt).toContain("<workflow>")
|
||||||
|
expect(athenaConfig.prompt).toContain("<synthesis_rules>")
|
||||||
|
expect(athenaConfig.prompt).toContain("<action_paths>")
|
||||||
|
expect(athenaConfig.prompt).toContain("<constraints>")
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#then places council setup after step 1 and removes first-action wording", () => {
|
||||||
|
const prompt = athenaConfig.prompt ?? ""
|
||||||
|
expect(prompt).not.toContain("CRITICAL: Council Setup (Your First Action)")
|
||||||
|
|
||||||
|
const step1Index = prompt.indexOf("Step 1: Understand the question and decide the route.")
|
||||||
|
const step2Index = prompt.indexOf("Step 2: Council setup (default flow before launch).")
|
||||||
|
|
||||||
|
expect(step1Index).toBeGreaterThan(-1)
|
||||||
|
expect(step2Index).toBeGreaterThan(step1Index)
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#then keeps edge-case disambiguation inside step 3 classification", () => {
|
||||||
|
const prompt = athenaConfig.prompt ?? ""
|
||||||
|
expect(prompt).toContain("Step 3: Classify the question intent.")
|
||||||
|
expect(prompt).toContain("Classification disambiguation rule:")
|
||||||
|
expect(prompt).not.toContain("Before selecting a route, apply this interpretation rule")
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#then requires question tool routing in self-answerable path", () => {
|
||||||
|
expect(athenaConfig.prompt).toContain("Athena MUST ask the user to choose direct answer vs council using the Question tool")
|
||||||
|
expect(athenaConfig.prompt).toContain("header: \"Routing\"")
|
||||||
|
expect(athenaConfig.prompt).toContain("label: \"Answer directly\"")
|
||||||
|
expect(athenaConfig.prompt).toContain("label: \"Consult council\"")
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#then requires question tool clarifications in ambiguous council path", () => {
|
||||||
|
expect(athenaConfig.prompt).toContain("Use the Question tool (not open-ended free text) for 1-2 targeted clarifications")
|
||||||
|
expect(athenaConfig.prompt).toContain("Formulate the questions and options dynamically")
|
||||||
|
expect(athenaConfig.prompt).not.toContain("header: \"Output Type\"")
|
||||||
|
expect(athenaConfig.prompt).not.toContain("header: \"Scope\"")
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#then keeps intent classification anchored to step 3 wording", () => {
|
||||||
|
expect(athenaConfig.prompt).toContain("Step 3: Classify the question intent.")
|
||||||
|
expect(athenaConfig.prompt).toContain("Then proceed to Step 2.")
|
||||||
|
expect(athenaConfig.prompt).not.toContain("Then classify intent and proceed to Step 2.")
|
||||||
|
expect(athenaConfig.prompt).not.toContain("Classify intent immediately and proceed to Step 2.")
|
||||||
|
})
|
||||||
|
|
||||||
|
it("#then excludes non-interactive mode branch from runtime prompt", () => {
|
||||||
|
expect(athenaConfig.prompt).not.toContain("Non-interactive mode (Question tool unavailable)")
|
||||||
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -63,7 +63,7 @@ You are conducting an **audit** — your goal is to find discrete issues, risks,
|
|||||||
- Severity determines priority: critical (blocks/breaks), high (significant risk), medium (should fix), low (nice to fix)
|
- Severity determines priority: critical (blocks/breaks), high (significant risk), medium (should fix), low (nice to fix)
|
||||||
- For each finding, provide the specific location (reference, section, or component where it occurs)
|
- For each finding, provide the specific location (reference, section, or component where it occurs)
|
||||||
- State your confidence: high (clear evidence), medium (likely but needs verification), low (suspicion, investigate further)
|
- State your confidence: high (clear evidence), medium (likely but needs verification), low (suspicion, investigate further)
|
||||||
- **This is a broad sweep, not a targeted trace.** If you are starting from a specific symptom and need to find its root cause, that is DIAGNOSE — not AUDIT.
|
- **This is a broad sweep, not a targeted trace.**.
|
||||||
|
|
||||||
**Required output fields per finding:**
|
**Required output fields per finding:**
|
||||||
- Title, Severity (critical/high/medium/low), Location, Confidence (high/medium/low)
|
- Title, Severity (critical/high/medium/low), Location, Confidence (high/medium/low)
|
||||||
|
|||||||
@@ -31,23 +31,7 @@ You MUST wrap your final analysis in <COUNCIL_MEMBER_RESPONSE> tags. This is how
|
|||||||
- Exploration logs and intermediate reasoning
|
- Exploration logs and intermediate reasoning
|
||||||
- Step-by-step search process
|
- Step-by-step search process
|
||||||
|
|
||||||
**Example structure:**
|
If you do not wrap your response in <COUNCIL_MEMBER_RESPONSE> tags, your analysis will not be included in the synthesis.`
|
||||||
\`\`\`
|
|
||||||
<COUNCIL_MEMBER_RESPONSE>
|
|
||||||
## Point 1: [Title]
|
|
||||||
- **Observation**: [what you found]
|
|
||||||
- **Confidence**: high
|
|
||||||
- **Details**: [supporting evidence or reasoning]
|
|
||||||
|
|
||||||
## Point 2: [Title]
|
|
||||||
...
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
[Overall assessment with confidence levels]
|
|
||||||
</COUNCIL_MEMBER_RESPONSE>
|
|
||||||
\`\`\`
|
|
||||||
|
|
||||||
If you do not wrap your response in these tags, your analysis will not be included in the synthesis.`
|
|
||||||
|
|
||||||
export const COUNCIL_SOLO_ADDENDUM = `
|
export const COUNCIL_SOLO_ADDENDUM = `
|
||||||
## Solo Analysis Mode
|
## Solo Analysis Mode
|
||||||
@@ -85,7 +69,9 @@ background_output(task_id="<id>")
|
|||||||
- Use \`librarian\` for documentation and external references
|
- Use \`librarian\` for documentation and external references
|
||||||
- Keep targeted file reads (Read tool) for yourself — delegate broad searches
|
- Keep targeted file reads (Read tool) for yourself — delegate broad searches
|
||||||
- Collect results with \`background_output\` after \`background_wait\` returns
|
- Collect results with \`background_output\` after \`background_wait\` returns
|
||||||
- Before generating your final \`<COUNCIL_MEMBER_RESPONSE>\`, cancel any remaining pending tasks with \`background_cancel\``
|
- Before generating your final \`<COUNCIL_MEMBER_RESPONSE>\`, wait for all the background tasks to finish.
|
||||||
|
- If you decide to form your final response before background tasks finishes, cancel any remaining pending tasks with \`background_cancel\`
|
||||||
|
`
|
||||||
|
|
||||||
export function createCouncilMemberAgent(model: string): AgentConfig {
|
export function createCouncilMemberAgent(model: string): AgentConfig {
|
||||||
// Allow-list: only read-only analysis tools + optional delegation.
|
// Allow-list: only read-only analysis tools + optional delegation.
|
||||||
|
|||||||
@@ -206,6 +206,7 @@ export async function createBuiltinAgents(
|
|||||||
const retryIfFinished = councilConfig.retry_failed_if_others_finished ?? false
|
const retryIfFinished = councilConfig.retry_failed_if_others_finished ?? false
|
||||||
const cancelOnQuorum = councilConfig.cancel_retrying_on_quorum ?? true
|
const cancelOnQuorum = councilConfig.cancel_retrying_on_quorum ?? true
|
||||||
const stuckThreshold = councilConfig.stuck_threshold_seconds ?? 120
|
const stuckThreshold = councilConfig.stuck_threshold_seconds ?? 120
|
||||||
|
const memberMaxRunning = councilConfig.member_max_running_seconds ?? 1800
|
||||||
|
|
||||||
let athenaPrompt = (result["athena"].prompt ?? "") + councilTaskInstructions
|
let athenaPrompt = (result["athena"].prompt ?? "") + councilTaskInstructions
|
||||||
athenaPrompt = athenaPrompt
|
athenaPrompt = athenaPrompt
|
||||||
@@ -213,7 +214,8 @@ export async function createBuiltinAgents(
|
|||||||
.replace(/\{RETRY_FAILED_IF_OTHERS_FINISHED\}/g, String(retryIfFinished))
|
.replace(/\{RETRY_FAILED_IF_OTHERS_FINISHED\}/g, String(retryIfFinished))
|
||||||
.replace(/\{CANCEL_RETRYING_ON_QUORUM\}/g, String(cancelOnQuorum))
|
.replace(/\{CANCEL_RETRYING_ON_QUORUM\}/g, String(cancelOnQuorum))
|
||||||
.replace(/\{STUCK_THRESHOLD_SECONDS\}/g, String(stuckThreshold))
|
.replace(/\{STUCK_THRESHOLD_SECONDS\}/g, String(stuckThreshold))
|
||||||
athenaPrompt += `\n\n## Council Resilience Config\n- retry_on_fail: ${retryOnFail}\n- retry_failed_if_others_finished: ${retryIfFinished}\n- cancel_retrying_on_quorum: ${cancelOnQuorum}\n- stuck_threshold_seconds: ${stuckThreshold}`
|
.replace(/\{MEMBER_MAX_RUNNING_SECONDS\}/g, String(memberMaxRunning))
|
||||||
|
athenaPrompt += `\n\n## Council Resilience Config\n- retry_on_fail: ${retryOnFail}\n- retry_failed_if_others_finished: ${retryIfFinished}\n- cancel_retrying_on_quorum: ${cancelOnQuorum}\n- stuck_threshold_seconds: ${stuckThreshold}\n- member_max_running_seconds: ${memberMaxRunning}`
|
||||||
|
|
||||||
result["athena"] = {
|
result["athena"] = {
|
||||||
...result["athena"],
|
...result["athena"],
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ describe("council-member-agents", () => {
|
|||||||
retry_failed_if_others_finished: false,
|
retry_failed_if_others_finished: false,
|
||||||
cancel_retrying_on_quorum: true,
|
cancel_retrying_on_quorum: true,
|
||||||
stuck_threshold_seconds: 120,
|
stuck_threshold_seconds: 120,
|
||||||
|
member_max_running_seconds: 1800,
|
||||||
}
|
}
|
||||||
//#when
|
//#when
|
||||||
const result = registerCouncilMemberAgents(config)
|
const result = registerCouncilMemberAgents(config)
|
||||||
@@ -32,6 +33,7 @@ describe("council-member-agents", () => {
|
|||||||
retry_failed_if_others_finished: false,
|
retry_failed_if_others_finished: false,
|
||||||
cancel_retrying_on_quorum: true,
|
cancel_retrying_on_quorum: true,
|
||||||
stuck_threshold_seconds: 120,
|
stuck_threshold_seconds: 120,
|
||||||
|
member_max_running_seconds: 1800,
|
||||||
}
|
}
|
||||||
//#when
|
//#when
|
||||||
const result = registerCouncilMemberAgents(config)
|
const result = registerCouncilMemberAgents(config)
|
||||||
@@ -52,6 +54,7 @@ describe("council-member-agents", () => {
|
|||||||
retry_failed_if_others_finished: false,
|
retry_failed_if_others_finished: false,
|
||||||
cancel_retrying_on_quorum: true,
|
cancel_retrying_on_quorum: true,
|
||||||
stuck_threshold_seconds: 120,
|
stuck_threshold_seconds: 120,
|
||||||
|
member_max_running_seconds: 1800,
|
||||||
}
|
}
|
||||||
//#when
|
//#when
|
||||||
const result = registerCouncilMemberAgents(config)
|
const result = registerCouncilMemberAgents(config)
|
||||||
@@ -72,6 +75,7 @@ describe("council-member-agents", () => {
|
|||||||
retry_failed_if_others_finished: false,
|
retry_failed_if_others_finished: false,
|
||||||
cancel_retrying_on_quorum: true,
|
cancel_retrying_on_quorum: true,
|
||||||
stuck_threshold_seconds: 120,
|
stuck_threshold_seconds: 120,
|
||||||
|
member_max_running_seconds: 1800,
|
||||||
}
|
}
|
||||||
//#when
|
//#when
|
||||||
const result = registerCouncilMemberAgents(config)
|
const result = registerCouncilMemberAgents(config)
|
||||||
|
|||||||
@@ -469,6 +469,14 @@ describe("CouncilConfigSchema — resilience fields", () => {
|
|||||||
expect(result.data.stuck_threshold_seconds).toBe(120)
|
expect(result.data.stuck_threshold_seconds).toBe(120)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it("#then applies default member_max_running_seconds of 1800", () => {
|
||||||
|
const result = CouncilConfigSchema.safeParse({ members: validMembers })
|
||||||
|
expect(result.success).toBe(true)
|
||||||
|
if (result.success) {
|
||||||
|
expect(result.data.member_max_running_seconds).toBe(1800)
|
||||||
|
}
|
||||||
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -481,6 +489,7 @@ describe("CouncilConfigSchema — resilience fields", () => {
|
|||||||
retry_failed_if_others_finished: true,
|
retry_failed_if_others_finished: true,
|
||||||
cancel_retrying_on_quorum: false,
|
cancel_retrying_on_quorum: false,
|
||||||
stuck_threshold_seconds: 60,
|
stuck_threshold_seconds: 60,
|
||||||
|
member_max_running_seconds: 2400,
|
||||||
}
|
}
|
||||||
const result = CouncilConfigSchema.safeParse(config)
|
const result = CouncilConfigSchema.safeParse(config)
|
||||||
expect(result.success).toBe(true)
|
expect(result.success).toBe(true)
|
||||||
@@ -489,6 +498,7 @@ describe("CouncilConfigSchema — resilience fields", () => {
|
|||||||
expect(result.data.retry_failed_if_others_finished).toBe(true)
|
expect(result.data.retry_failed_if_others_finished).toBe(true)
|
||||||
expect(result.data.cancel_retrying_on_quorum).toBe(false)
|
expect(result.data.cancel_retrying_on_quorum).toBe(false)
|
||||||
expect(result.data.stuck_threshold_seconds).toBe(60)
|
expect(result.data.stuck_threshold_seconds).toBe(60)
|
||||||
|
expect(result.data.member_max_running_seconds).toBe(2400)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
@@ -521,6 +531,15 @@ describe("CouncilConfigSchema — resilience fields", () => {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe("#given member_max_running_seconds below minimum", () => {
|
||||||
|
describe("#when parsed with 30", () => {
|
||||||
|
it("#then fails validation", () => {
|
||||||
|
const result = CouncilConfigSchema.safeParse({ members: validMembers, member_max_running_seconds: 30 })
|
||||||
|
expect(result.success).toBe(false)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
describe("#given backward-compatible config with only members", () => {
|
describe("#given backward-compatible config with only members", () => {
|
||||||
describe("#when parsed", () => {
|
describe("#when parsed", () => {
|
||||||
it("#then succeeds without errors — new fields are optional", () => {
|
it("#then succeeds without errors — new fields are optional", () => {
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ export const CouncilConfigSchema = z.object({
|
|||||||
retry_failed_if_others_finished: z.boolean().default(false),
|
retry_failed_if_others_finished: z.boolean().default(false),
|
||||||
cancel_retrying_on_quorum: z.boolean().default(true),
|
cancel_retrying_on_quorum: z.boolean().default(true),
|
||||||
stuck_threshold_seconds: z.number().min(30).default(120),
|
stuck_threshold_seconds: z.number().min(30).default(120),
|
||||||
|
member_max_running_seconds: z.number().min(60).default(1800),
|
||||||
}).strict()
|
}).strict()
|
||||||
|
|
||||||
export type CouncilMemberConfig = z.infer<typeof CouncilMemberSchema>
|
export type CouncilMemberConfig = z.infer<typeof CouncilMemberSchema>
|
||||||
|
|||||||
Reference in New Issue
Block a user