refactor(librarian): teach proper ast_grep_search discipline
ast_grep_search was mentioned exactly once in the librarian prompt (bundled as 'grep/ast_grep_search for function/class') with no syntax guidance. When the librarian cloned a repo and tried to match code shape, it fell into the same regex-in-AST trap as the main agent. Two targeted edits, no rewrite of the surrounding request-classification flow: - Phase 1 TYPE B 'Find the implementation' now separates ast_grep_search (code shape) from grep (text/literals) and reminds the LLM that AST patterns use $VAR and $$$ and are not regex. - TOOL REFERENCE adds a dedicated ast_grep_search row with valid examples and the explicit regex anti-pattern list, alongside tightened guidance for grep_app and grep so the LLM picks the right tool for cross-repo vs single-repo, text vs shape.
This commit is contained in:
@@ -0,0 +1,84 @@
|
||||
/// <reference types="bun-types" />
|
||||
|
||||
import { describe, expect, it } from "bun:test"
|
||||
import { createLibrarianAgent } from "./librarian"
|
||||
|
||||
describe("librarian agent ast-grep discipline", () => {
|
||||
const model = "openai/gpt-5.4-mini-fast"
|
||||
|
||||
it("#given the prompt #when inspecting TOOL REFERENCE #then documents ast_grep_search", () => {
|
||||
// given
|
||||
const agent = createLibrarianAgent(model)
|
||||
|
||||
// when
|
||||
const prompt = agent.prompt ?? ""
|
||||
|
||||
// then
|
||||
expect(prompt).toContain("ast_grep_search")
|
||||
expect(prompt).toContain("$$$")
|
||||
expect(prompt).toContain("function $NAME($$$) { $$$ }")
|
||||
})
|
||||
|
||||
it("#given the prompt #when inspecting #then warns against regex inside ast_grep_search", () => {
|
||||
// given
|
||||
const agent = createLibrarianAgent(model)
|
||||
|
||||
// when
|
||||
const prompt = agent.prompt ?? ""
|
||||
|
||||
// then
|
||||
expect(prompt.toLowerCase()).toContain("not regex")
|
||||
expect(prompt).toContain("|")
|
||||
expect(prompt).toContain(".*")
|
||||
expect(prompt).toContain("\\w")
|
||||
})
|
||||
|
||||
it("#given the prompt #when inspecting #then directs LLM to grep/grep_app for text search", () => {
|
||||
// given
|
||||
const agent = createLibrarianAgent(model)
|
||||
|
||||
// when
|
||||
const prompt = agent.prompt ?? ""
|
||||
|
||||
// then
|
||||
expect(prompt.toLowerCase()).toContain("for text")
|
||||
expect(prompt).toContain("grep_app")
|
||||
})
|
||||
|
||||
it("#given the prompt #when inspecting Implementation phase #then recommends ast_grep_search for code shape", () => {
|
||||
// given
|
||||
const agent = createLibrarianAgent(model)
|
||||
|
||||
// when
|
||||
const prompt = agent.prompt ?? ""
|
||||
|
||||
// then
|
||||
expect(prompt).toContain("ast_grep_search for code shape")
|
||||
})
|
||||
|
||||
it("#given the prompt #when inspecting #then preserves the evidence + permalink contract", () => {
|
||||
// given
|
||||
const agent = createLibrarianAgent(model)
|
||||
|
||||
// when
|
||||
const prompt = agent.prompt ?? ""
|
||||
|
||||
// then
|
||||
expect(prompt).toContain("GitHub permalinks")
|
||||
expect(prompt).toContain("MANDATORY CITATION FORMAT")
|
||||
})
|
||||
|
||||
it("#given the prompt #when inspecting #then preserves request classification phases", () => {
|
||||
// given
|
||||
const agent = createLibrarianAgent(model)
|
||||
|
||||
// when
|
||||
const prompt = agent.prompt ?? ""
|
||||
|
||||
// then
|
||||
expect(prompt).toContain("TYPE A: CONCEPTUAL")
|
||||
expect(prompt).toContain("TYPE B: IMPLEMENTATION")
|
||||
expect(prompt).toContain("TYPE C: CONTEXT")
|
||||
expect(prompt).toContain("TYPE D: COMPREHENSIVE")
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user