add skills
This commit is contained in:
YeonGyu-Kim
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"run_id": "eval-5-without_skill",
|
||||
"expectations": [
|
||||
{"text": "Plan uses git worktree in a sibling directory", "passed": false, "evidence": "git checkout -b, no worktree"},
|
||||
{"text": "References actual comment-checker hook files", "passed": true, "evidence": "Deep analysis of Go binary, tree-sitter, formatter.go, agent_memo.go with line numbers"},
|
||||
{"text": "Adds test cases for Note: false positive scenarios", "passed": true, "evidence": "Detailed test cases distinguishing legit vs AI slop patterns"},
|
||||
{"text": "Verification loop includes all 3 gates", "passed": false, "evidence": "Only bun test and typecheck. No review-work or Cubic."},
|
||||
{"text": "Only modifies regex and adds tests — no unrelated changes", "passed": true, "evidence": "Adds allowed-prefix filter module — focused approach with config extension"}
|
||||
]
|
||||
}
|
||||
+529
@@ -0,0 +1,529 @@
|
||||
# Code Changes: comment-checker false positive fix
|
||||
|
||||
## Change 1: Extend config schema
|
||||
|
||||
**File: `src/config/schema/comment-checker.ts`**
|
||||
|
||||
```typescript
|
||||
// BEFORE
|
||||
import { z } from "zod"
|
||||
|
||||
export const CommentCheckerConfigSchema = z.object({
|
||||
/** Custom prompt to replace the default warning message. Use {{comments}} placeholder for detected comments XML. */
|
||||
custom_prompt: z.string().optional(),
|
||||
})
|
||||
|
||||
export type CommentCheckerConfig = z.infer<typeof CommentCheckerConfigSchema>
|
||||
```
|
||||
|
||||
```typescript
|
||||
// AFTER
|
||||
import { z } from "zod"
|
||||
|
||||
const DEFAULT_ALLOWED_COMMENT_PREFIXES = [
|
||||
"note:",
|
||||
"todo:",
|
||||
"fixme:",
|
||||
"hack:",
|
||||
"xxx:",
|
||||
"warning:",
|
||||
"important:",
|
||||
"bug:",
|
||||
"optimize:",
|
||||
"workaround:",
|
||||
"safety:",
|
||||
"security:",
|
||||
"perf:",
|
||||
"see:",
|
||||
"ref:",
|
||||
"cf.",
|
||||
]
|
||||
|
||||
export const CommentCheckerConfigSchema = z.object({
|
||||
/** Custom prompt to replace the default warning message. Use {{comments}} placeholder for detected comments XML. */
|
||||
custom_prompt: z.string().optional(),
|
||||
/** Comment prefixes considered legitimate (not AI slop). Case-insensitive. Defaults include Note:, TODO:, FIXME:, etc. */
|
||||
allowed_comment_prefixes: z.array(z.string()).optional().default(DEFAULT_ALLOWED_COMMENT_PREFIXES),
|
||||
})
|
||||
|
||||
export type CommentCheckerConfig = z.infer<typeof CommentCheckerConfigSchema>
|
||||
```
|
||||
|
||||
## Change 2: Create allowed-prefix-filter module
|
||||
|
||||
**File: `src/hooks/comment-checker/allowed-prefix-filter.ts`** (NEW)
|
||||
|
||||
```typescript
|
||||
const COMMENT_XML_REGEX = /<comment\s+line-number="\d+">([\s\S]*?)<\/comment>/g
|
||||
const COMMENTS_BLOCK_REGEX = /<comments\s+file="[^"]*">\s*([\s\S]*?)\s*<\/comments>/g
|
||||
const AGENT_MEMO_HEADER_REGEX = /🚨 AGENT MEMO COMMENT DETECTED.*?---\n\n/s
|
||||
|
||||
function stripCommentPrefix(text: string): string {
|
||||
let stripped = text.trim()
|
||||
for (const prefix of ["//", "#", "/*", "--", "*"]) {
|
||||
if (stripped.startsWith(prefix)) {
|
||||
stripped = stripped.slice(prefix.length).trim()
|
||||
break
|
||||
}
|
||||
}
|
||||
return stripped
|
||||
}
|
||||
|
||||
function isAllowedComment(commentText: string, allowedPrefixes: string[]): boolean {
|
||||
const stripped = stripCommentPrefix(commentText).toLowerCase()
|
||||
return allowedPrefixes.some((prefix) => stripped.startsWith(prefix.toLowerCase()))
|
||||
}
|
||||
|
||||
function extractCommentTexts(xmlBlock: string): string[] {
|
||||
const texts: string[] = []
|
||||
let match: RegExpExecArray | null
|
||||
const regex = new RegExp(COMMENT_XML_REGEX.source, COMMENT_XML_REGEX.flags)
|
||||
while ((match = regex.exec(xmlBlock)) !== null) {
|
||||
texts.push(match[1])
|
||||
}
|
||||
return texts
|
||||
}
|
||||
|
||||
export function filterAllowedComments(
|
||||
message: string,
|
||||
allowedPrefixes: string[],
|
||||
): { hasRemainingComments: boolean; filteredMessage: string } {
|
||||
if (!message || allowedPrefixes.length === 0) {
|
||||
return { hasRemainingComments: true, filteredMessage: message }
|
||||
}
|
||||
|
||||
const commentTexts = extractCommentTexts(message)
|
||||
|
||||
if (commentTexts.length === 0) {
|
||||
return { hasRemainingComments: true, filteredMessage: message }
|
||||
}
|
||||
|
||||
const disallowedComments = commentTexts.filter(
|
||||
(text) => !isAllowedComment(text, allowedPrefixes),
|
||||
)
|
||||
|
||||
if (disallowedComments.length === 0) {
|
||||
return { hasRemainingComments: false, filteredMessage: "" }
|
||||
}
|
||||
|
||||
if (disallowedComments.length === commentTexts.length) {
|
||||
return { hasRemainingComments: true, filteredMessage: message }
|
||||
}
|
||||
|
||||
let filteredMessage = message
|
||||
for (const text of commentTexts) {
|
||||
if (isAllowedComment(text, allowedPrefixes)) {
|
||||
const escapedText = text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
|
||||
const lineRegex = new RegExp(`\\s*<comment\\s+line-number="\\d+">${escapedText}</comment>\\n?`, "g")
|
||||
filteredMessage = filteredMessage.replace(lineRegex, "")
|
||||
}
|
||||
}
|
||||
|
||||
filteredMessage = filteredMessage.replace(AGENT_MEMO_HEADER_REGEX, "")
|
||||
|
||||
return { hasRemainingComments: true, filteredMessage }
|
||||
}
|
||||
```
|
||||
|
||||
## Change 3: Thread config through cli-runner.ts
|
||||
|
||||
**File: `src/hooks/comment-checker/cli-runner.ts`**
|
||||
|
||||
```typescript
|
||||
// BEFORE (processWithCli signature and body)
|
||||
export async function processWithCli(
|
||||
input: { tool: string; sessionID: string; callID: string },
|
||||
pendingCall: PendingCall,
|
||||
output: { output: string },
|
||||
cliPath: string,
|
||||
customPrompt: string | undefined,
|
||||
debugLog: (...args: unknown[]) => void,
|
||||
): Promise<void> {
|
||||
await withCommentCheckerLock(async () => {
|
||||
// ...
|
||||
const result = await runCommentChecker(hookInput, cliPath, customPrompt)
|
||||
if (result.hasComments && result.message) {
|
||||
debugLog("CLI detected comments, appending message")
|
||||
output.output += `\n\n${result.message}`
|
||||
} else {
|
||||
debugLog("CLI: no comments detected")
|
||||
}
|
||||
}, undefined, debugLog)
|
||||
}
|
||||
```
|
||||
|
||||
```typescript
|
||||
// AFTER
|
||||
import { filterAllowedComments } from "./allowed-prefix-filter"
|
||||
|
||||
export async function processWithCli(
|
||||
input: { tool: string; sessionID: string; callID: string },
|
||||
pendingCall: PendingCall,
|
||||
output: { output: string },
|
||||
cliPath: string,
|
||||
customPrompt: string | undefined,
|
||||
allowedPrefixes: string[],
|
||||
debugLog: (...args: unknown[]) => void,
|
||||
): Promise<void> {
|
||||
await withCommentCheckerLock(async () => {
|
||||
void input
|
||||
debugLog("using CLI mode with path:", cliPath)
|
||||
|
||||
const hookInput: HookInput = {
|
||||
session_id: pendingCall.sessionID,
|
||||
tool_name: pendingCall.tool.charAt(0).toUpperCase() + pendingCall.tool.slice(1),
|
||||
transcript_path: "",
|
||||
cwd: process.cwd(),
|
||||
hook_event_name: "PostToolUse",
|
||||
tool_input: {
|
||||
file_path: pendingCall.filePath,
|
||||
content: pendingCall.content,
|
||||
old_string: pendingCall.oldString,
|
||||
new_string: pendingCall.newString,
|
||||
edits: pendingCall.edits,
|
||||
},
|
||||
}
|
||||
|
||||
const result = await runCommentChecker(hookInput, cliPath, customPrompt)
|
||||
|
||||
if (result.hasComments && result.message) {
|
||||
const { hasRemainingComments, filteredMessage } = filterAllowedComments(
|
||||
result.message,
|
||||
allowedPrefixes,
|
||||
)
|
||||
if (hasRemainingComments && filteredMessage) {
|
||||
debugLog("CLI detected comments, appending filtered message")
|
||||
output.output += `\n\n${filteredMessage}`
|
||||
} else {
|
||||
debugLog("CLI: all detected comments matched allowed prefixes, suppressing")
|
||||
}
|
||||
} else {
|
||||
debugLog("CLI: no comments detected")
|
||||
}
|
||||
}, undefined, debugLog)
|
||||
}
|
||||
|
||||
// Same change applied to processApplyPatchEditsWithCli - add allowedPrefixes parameter
|
||||
export async function processApplyPatchEditsWithCli(
|
||||
sessionID: string,
|
||||
edits: ApplyPatchEdit[],
|
||||
output: { output: string },
|
||||
cliPath: string,
|
||||
customPrompt: string | undefined,
|
||||
allowedPrefixes: string[],
|
||||
debugLog: (...args: unknown[]) => void,
|
||||
): Promise<void> {
|
||||
debugLog("processing apply_patch edits:", edits.length)
|
||||
|
||||
for (const edit of edits) {
|
||||
await withCommentCheckerLock(async () => {
|
||||
const hookInput: HookInput = {
|
||||
session_id: sessionID,
|
||||
tool_name: "Edit",
|
||||
transcript_path: "",
|
||||
cwd: process.cwd(),
|
||||
hook_event_name: "PostToolUse",
|
||||
tool_input: {
|
||||
file_path: edit.filePath,
|
||||
old_string: edit.before,
|
||||
new_string: edit.after,
|
||||
},
|
||||
}
|
||||
|
||||
const result = await runCommentChecker(hookInput, cliPath, customPrompt)
|
||||
|
||||
if (result.hasComments && result.message) {
|
||||
const { hasRemainingComments, filteredMessage } = filterAllowedComments(
|
||||
result.message,
|
||||
allowedPrefixes,
|
||||
)
|
||||
if (hasRemainingComments && filteredMessage) {
|
||||
debugLog("CLI detected comments for apply_patch file:", edit.filePath)
|
||||
output.output += `\n\n${filteredMessage}`
|
||||
}
|
||||
}
|
||||
}, undefined, debugLog)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Change 4: Update hook.ts to pass config
|
||||
|
||||
**File: `src/hooks/comment-checker/hook.ts`**
|
||||
|
||||
```typescript
|
||||
// BEFORE (in tool.execute.after handler, around line 177)
|
||||
await processWithCli(input, pendingCall, output, cliPath, config?.custom_prompt, debugLog)
|
||||
|
||||
// AFTER
|
||||
const allowedPrefixes = config?.allowed_comment_prefixes ?? []
|
||||
await processWithCli(input, pendingCall, output, cliPath, config?.custom_prompt, allowedPrefixes, debugLog)
|
||||
```
|
||||
|
||||
```typescript
|
||||
// BEFORE (in apply_patch section, around line 147-154)
|
||||
await processApplyPatchEditsWithCli(
|
||||
input.sessionID,
|
||||
edits,
|
||||
output,
|
||||
cliPath,
|
||||
config?.custom_prompt,
|
||||
debugLog,
|
||||
)
|
||||
|
||||
// AFTER
|
||||
const allowedPrefixes = config?.allowed_comment_prefixes ?? []
|
||||
await processApplyPatchEditsWithCli(
|
||||
input.sessionID,
|
||||
edits,
|
||||
output,
|
||||
cliPath,
|
||||
config?.custom_prompt,
|
||||
allowedPrefixes,
|
||||
debugLog,
|
||||
)
|
||||
```
|
||||
|
||||
## Change 5: Test file for allowed-prefix-filter
|
||||
|
||||
**File: `src/hooks/comment-checker/allowed-prefix-filter.test.ts`** (NEW)
|
||||
|
||||
```typescript
|
||||
import { describe, test, expect } from "bun:test"
|
||||
|
||||
import { filterAllowedComments } from "./allowed-prefix-filter"
|
||||
|
||||
const DEFAULT_PREFIXES = [
|
||||
"note:", "todo:", "fixme:", "hack:", "xxx:", "warning:",
|
||||
"important:", "bug:", "optimize:", "workaround:", "safety:",
|
||||
"security:", "perf:", "see:", "ref:", "cf.",
|
||||
]
|
||||
|
||||
function buildMessage(comments: { line: number; text: string }[], filePath = "/tmp/test.ts"): string {
|
||||
const xml = comments
|
||||
.map((c) => `\t<comment line-number="${c.line}">${c.text}</comment>`)
|
||||
.join("\n")
|
||||
return `COMMENT/DOCSTRING DETECTED - IMMEDIATE ACTION REQUIRED\n\n` +
|
||||
`Your recent changes contain comments or docstrings, which triggered this hook.\n` +
|
||||
`Detected comments/docstrings:\n` +
|
||||
`<comments file="${filePath}">\n${xml}\n</comments>\n`
|
||||
}
|
||||
|
||||
describe("allowed-prefix-filter", () => {
|
||||
describe("#given default allowed prefixes", () => {
|
||||
describe("#when message contains only Note: comments", () => {
|
||||
test("#then should suppress the entire message", () => {
|
||||
const message = buildMessage([
|
||||
{ line: 5, text: "// Note: Thread-safe implementation" },
|
||||
{ line: 12, text: "// NOTE: See RFC 7231 for details" },
|
||||
])
|
||||
|
||||
const result = filterAllowedComments(message, DEFAULT_PREFIXES)
|
||||
|
||||
expect(result.hasRemainingComments).toBe(false)
|
||||
expect(result.filteredMessage).toBe("")
|
||||
})
|
||||
})
|
||||
|
||||
describe("#when message contains only TODO/FIXME comments", () => {
|
||||
test("#then should suppress the entire message", () => {
|
||||
const message = buildMessage([
|
||||
{ line: 3, text: "// TODO: implement caching" },
|
||||
{ line: 7, text: "// FIXME: race condition here" },
|
||||
{ line: 15, text: "# HACK: workaround for upstream bug" },
|
||||
])
|
||||
|
||||
const result = filterAllowedComments(message, DEFAULT_PREFIXES)
|
||||
|
||||
expect(result.hasRemainingComments).toBe(false)
|
||||
expect(result.filteredMessage).toBe("")
|
||||
})
|
||||
})
|
||||
|
||||
describe("#when message contains only AI slop comments", () => {
|
||||
test("#then should keep the entire message", () => {
|
||||
const message = buildMessage([
|
||||
{ line: 2, text: "// Added new validation logic" },
|
||||
{ line: 8, text: "// Refactored for better performance" },
|
||||
])
|
||||
|
||||
const result = filterAllowedComments(message, DEFAULT_PREFIXES)
|
||||
|
||||
expect(result.hasRemainingComments).toBe(true)
|
||||
expect(result.filteredMessage).toBe(message)
|
||||
})
|
||||
})
|
||||
|
||||
describe("#when message contains mix of legitimate and slop comments", () => {
|
||||
test("#then should keep message but remove allowed comment XML entries", () => {
|
||||
const message = buildMessage([
|
||||
{ line: 5, text: "// Note: Thread-safe implementation" },
|
||||
{ line: 10, text: "// Changed from old API to new API" },
|
||||
])
|
||||
|
||||
const result = filterAllowedComments(message, DEFAULT_PREFIXES)
|
||||
|
||||
expect(result.hasRemainingComments).toBe(true)
|
||||
expect(result.filteredMessage).not.toContain("Thread-safe implementation")
|
||||
expect(result.filteredMessage).toContain("Changed from old API to new API")
|
||||
})
|
||||
})
|
||||
|
||||
describe("#when Note: comment has lowercase prefix", () => {
|
||||
test("#then should still be treated as allowed (case-insensitive)", () => {
|
||||
const message = buildMessage([
|
||||
{ line: 1, text: "// note: this is case insensitive" },
|
||||
])
|
||||
|
||||
const result = filterAllowedComments(message, DEFAULT_PREFIXES)
|
||||
|
||||
expect(result.hasRemainingComments).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe("#when comment uses hash prefix", () => {
|
||||
test("#then should strip prefix before matching", () => {
|
||||
const message = buildMessage([
|
||||
{ line: 1, text: "# Note: Python style comment" },
|
||||
{ line: 5, text: "# TODO: something to do" },
|
||||
])
|
||||
|
||||
const result = filterAllowedComments(message, DEFAULT_PREFIXES)
|
||||
|
||||
expect(result.hasRemainingComments).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe("#when comment has Security: prefix", () => {
|
||||
test("#then should be treated as allowed", () => {
|
||||
const message = buildMessage([
|
||||
{ line: 1, text: "// Security: validate input before processing" },
|
||||
])
|
||||
|
||||
const result = filterAllowedComments(message, DEFAULT_PREFIXES)
|
||||
|
||||
expect(result.hasRemainingComments).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe("#when comment has Warning: prefix", () => {
|
||||
test("#then should be treated as allowed", () => {
|
||||
const message = buildMessage([
|
||||
{ line: 1, text: "// WARNING: This mutates the input array" },
|
||||
])
|
||||
|
||||
const result = filterAllowedComments(message, DEFAULT_PREFIXES)
|
||||
|
||||
expect(result.hasRemainingComments).toBe(false)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe("#given empty allowed prefixes", () => {
|
||||
describe("#when any comments are detected", () => {
|
||||
test("#then should pass through unfiltered", () => {
|
||||
const message = buildMessage([
|
||||
{ line: 1, text: "// Note: this should pass through" },
|
||||
])
|
||||
|
||||
const result = filterAllowedComments(message, [])
|
||||
|
||||
expect(result.hasRemainingComments).toBe(true)
|
||||
expect(result.filteredMessage).toBe(message)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe("#given custom allowed prefixes", () => {
|
||||
describe("#when comment matches custom prefix", () => {
|
||||
test("#then should suppress it", () => {
|
||||
const message = buildMessage([
|
||||
{ line: 1, text: "// PERF: O(n log n) complexity" },
|
||||
])
|
||||
|
||||
const result = filterAllowedComments(message, ["perf:"])
|
||||
|
||||
expect(result.hasRemainingComments).toBe(false)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe("#given empty message", () => {
|
||||
describe("#when filterAllowedComments is called", () => {
|
||||
test("#then should return hasRemainingComments true with empty string", () => {
|
||||
const result = filterAllowedComments("", DEFAULT_PREFIXES)
|
||||
|
||||
expect(result.hasRemainingComments).toBe(true)
|
||||
expect(result.filteredMessage).toBe("")
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe("#given message with agent memo header", () => {
|
||||
describe("#when all flagged comments are legitimate Note: comments", () => {
|
||||
test("#then should suppress agent memo header along with comments", () => {
|
||||
const message =
|
||||
"🚨 AGENT MEMO COMMENT DETECTED - CODE SMELL ALERT 🚨\n\n" +
|
||||
"⚠️ AGENT MEMO COMMENTS DETECTED - THIS IS A CODE SMELL ⚠️\n\n" +
|
||||
"You left \"memo-style\" comments...\n\n---\n\n" +
|
||||
"Your recent changes contain comments...\n" +
|
||||
"Detected comments/docstrings:\n" +
|
||||
'<comments file="/tmp/test.ts">\n' +
|
||||
'\t<comment line-number="5">// Note: Thread-safe</comment>\n' +
|
||||
"</comments>\n"
|
||||
|
||||
const result = filterAllowedComments(message, DEFAULT_PREFIXES)
|
||||
|
||||
expect(result.hasRemainingComments).toBe(false)
|
||||
expect(result.filteredMessage).toBe("")
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
```
|
||||
|
||||
## Change 6: Update existing test for new parameter
|
||||
|
||||
**File: `src/hooks/comment-checker/hook.apply-patch.test.ts`**
|
||||
|
||||
The `processApplyPatchEditsWithCli` mock needs to account for the new `allowedPrefixes` parameter:
|
||||
|
||||
```typescript
|
||||
// BEFORE (line 58)
|
||||
expect(processApplyPatchEditsWithCli).toHaveBeenCalledWith(
|
||||
"ses_test",
|
||||
[
|
||||
{ filePath: "/repo/src/a.ts", before: "const a = 1\n", after: "// comment\nconst a = 1\n" },
|
||||
{ filePath: "/repo/src/new.ts", before: "const b = 1\n", after: "// moved comment\nconst b = 1\n" },
|
||||
],
|
||||
expect.any(Object),
|
||||
"/tmp/fake-comment-checker",
|
||||
undefined,
|
||||
expect.any(Function),
|
||||
)
|
||||
|
||||
// AFTER - add allowed_comment_prefixes argument
|
||||
expect(processApplyPatchEditsWithCli).toHaveBeenCalledWith(
|
||||
"ses_test",
|
||||
[
|
||||
{ filePath: "/repo/src/a.ts", before: "const a = 1\n", after: "// comment\nconst a = 1\n" },
|
||||
{ filePath: "/repo/src/new.ts", before: "const b = 1\n", after: "// moved comment\nconst b = 1\n" },
|
||||
],
|
||||
expect.any(Object),
|
||||
"/tmp/fake-comment-checker",
|
||||
undefined,
|
||||
expect.any(Array),
|
||||
expect.any(Function),
|
||||
)
|
||||
```
|
||||
|
||||
## Summary of all touched files
|
||||
|
||||
| File | Action | Description |
|
||||
|------|--------|-------------|
|
||||
| `src/config/schema/comment-checker.ts` | Modified | Add `allowed_comment_prefixes` with defaults |
|
||||
| `src/hooks/comment-checker/allowed-prefix-filter.ts` | **New** | Post-processing filter for legitimate comment prefixes |
|
||||
| `src/hooks/comment-checker/allowed-prefix-filter.test.ts` | **New** | 11 test cases covering false positives and edge cases |
|
||||
| `src/hooks/comment-checker/cli-runner.ts` | Modified | Thread `allowedPrefixes` param, apply filter after binary result |
|
||||
| `src/hooks/comment-checker/hook.ts` | Modified | Pass `allowed_comment_prefixes` from config to CLI runner |
|
||||
| `src/hooks/comment-checker/hook.apply-patch.test.ts` | Modified | Update mock assertions for new parameter |
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
# Execution Plan: Relax comment-checker hook false positives
|
||||
|
||||
## Problem Analysis
|
||||
|
||||
The comment-checker hook delegates to an external Go binary (`code-yeongyu/go-claude-code-comment-checker`). The binary:
|
||||
1. Detects ALL comments in written/edited code using tree-sitter
|
||||
2. Filters out only BDD markers, linter directives, and shebangs
|
||||
3. Flags every remaining comment as problematic (exit code 2)
|
||||
4. In the output formatter (`formatter.go`), uses `AgentMemoFilter` to categorize comments for display
|
||||
|
||||
The `AgentMemoFilter` in `pkg/filters/agent_memo.go` contains the overly aggressive regex:
|
||||
```go
|
||||
regexp.MustCompile(`(?i)^[\s#/*-]*note:\s*\w`),
|
||||
```
|
||||
|
||||
This matches ANY comment starting with `Note:` (case-insensitive) followed by a word character, causing legitimate comments like `// Note: Thread-safe implementation` or `// NOTE: See RFC 7231` to be classified as "AGENT MEMO" AI slop with an aggressive warning banner.
|
||||
|
||||
Additionally, the binary flags ALL non-filtered comments (not just agent memos), so even without the `Note:` regex, `// Note: ...` comments would still be flagged as generic "COMMENT DETECTED."
|
||||
|
||||
## Architecture Understanding
|
||||
|
||||
```
|
||||
TypeScript (oh-my-opencode) Go Binary (go-claude-code-comment-checker)
|
||||
───────────────────────────── ──────────────────────────────────────────
|
||||
hook.ts main.go
|
||||
├─ tool.execute.before ├─ Read JSON from stdin
|
||||
│ └─ registerPendingCall() ├─ Detect comments (tree-sitter)
|
||||
└─ tool.execute.after ├─ applyFilters (BDD, Directive, Shebang)
|
||||
└─ processWithCli() ├─ FormatHookMessage (uses AgentMemoFilter for display)
|
||||
└─ runCommentChecker() └─ exit 0 (clean) or exit 2 (comments found, message on stderr)
|
||||
└─ spawn binary, pipe JSON
|
||||
└─ read stderr → message
|
||||
└─ append to output
|
||||
```
|
||||
|
||||
Key files in oh-my-opencode:
|
||||
- `src/hooks/comment-checker/hook.ts` - Hook factory, registers before/after handlers
|
||||
- `src/hooks/comment-checker/cli-runner.ts` - Orchestrates CLI invocation, semaphore
|
||||
- `src/hooks/comment-checker/cli.ts` - Binary resolution, process spawning, timeout handling
|
||||
- `src/hooks/comment-checker/types.ts` - PendingCall, CommentInfo types
|
||||
- `src/config/schema/comment-checker.ts` - Config schema (currently only `custom_prompt`)
|
||||
|
||||
Key files in Go binary:
|
||||
- `pkg/filters/agent_memo.go` - Contains the aggressive `note:\s*\w` regex (line 20)
|
||||
- `pkg/output/formatter.go` - Uses AgentMemoFilter to add "AGENT MEMO" warnings
|
||||
- `cmd/comment-checker/main.go` - Filter pipeline (BDD + Directive + Shebang only)
|
||||
|
||||
## Step-by-Step Plan
|
||||
|
||||
### Step 1: Create feature branch
|
||||
```bash
|
||||
git checkout dev
|
||||
git pull origin dev
|
||||
git checkout -b fix/comment-checker-note-false-positive
|
||||
```
|
||||
|
||||
### Step 2: Extend CommentCheckerConfigSchema
|
||||
**File: `src/config/schema/comment-checker.ts`**
|
||||
|
||||
Add `allowed_comment_prefixes` field with sensible defaults. This lets users configure which comment prefixes should be treated as legitimate (not AI slop).
|
||||
|
||||
### Step 3: Add a post-processing filter in cli-runner.ts
|
||||
**File: `src/hooks/comment-checker/cli-runner.ts`**
|
||||
|
||||
After the Go binary returns its result, parse the stderr message to identify and suppress comments that match allowed prefixes. The binary's output contains XML like:
|
||||
```xml
|
||||
<comments file="/path/to/file.ts">
|
||||
<comment line-number="5">// Note: Thread-safe</comment>
|
||||
</comments>
|
||||
```
|
||||
|
||||
Add a function `filterAllowedComments()` that:
|
||||
1. Extracts `<comment>` elements from the message
|
||||
2. Checks if the comment text matches any allowed prefix pattern
|
||||
3. If ALL flagged comments match allowed patterns, suppress the entire warning
|
||||
4. If some comments are legitimate and some aren't, rebuild the message without the legitimate ones
|
||||
|
||||
### Step 4: Create dedicated filter module
|
||||
**File: `src/hooks/comment-checker/allowed-prefix-filter.ts`** (new)
|
||||
|
||||
Extract the filtering logic into its own module per the 200 LOC / single-responsibility rule.
|
||||
|
||||
### Step 5: Pass allowed_comment_prefixes through the hook chain
|
||||
**File: `src/hooks/comment-checker/hook.ts`**
|
||||
|
||||
Thread the `allowed_comment_prefixes` config from `createCommentCheckerHooks()` down to `processWithCli()` and `processApplyPatchEditsWithCli()`.
|
||||
|
||||
### Step 6: Add test cases
|
||||
**File: `src/hooks/comment-checker/allowed-prefix-filter.test.ts`** (new)
|
||||
|
||||
Test cases covering:
|
||||
- `// Note: Thread-safe implementation` - should NOT be flagged (false positive)
|
||||
- `// NOTE: See RFC 7231 for details` - should NOT be flagged
|
||||
- `// Note: changed from X to Y` - SHOULD still be flagged (genuine AI slop)
|
||||
- `// TODO: implement caching` - should NOT be flagged
|
||||
- `// FIXME: race condition` - should NOT be flagged
|
||||
- `// HACK: workaround for upstream bug` - should NOT be flagged
|
||||
- `// Added new validation logic` - SHOULD be flagged
|
||||
- Custom allowed patterns from config
|
||||
|
||||
**File: `src/hooks/comment-checker/cli-runner.test.ts`** (new or extend cli.test.ts)
|
||||
|
||||
Integration-level tests for the post-processing pipeline.
|
||||
|
||||
### Step 7: Verify
|
||||
```bash
|
||||
bun test src/hooks/comment-checker/
|
||||
bun run typecheck
|
||||
```
|
||||
|
||||
### Step 8: Commit and push
|
||||
```bash
|
||||
git add -A
|
||||
git commit -m "fix(comment-checker): add allowed-prefix filter to reduce false positives on Note: comments"
|
||||
git push -u origin fix/comment-checker-note-false-positive
|
||||
```
|
||||
|
||||
### Step 9: Create PR
|
||||
```bash
|
||||
gh pr create --title "fix(comment-checker): reduce false positives for legitimate Note: comments" --body-file /tmp/pr-body.md --base dev
|
||||
```
|
||||
|
||||
### Step 10 (Follow-up): Upstream Go binary fix
|
||||
File an issue or PR on `code-yeongyu/go-claude-code-comment-checker` to:
|
||||
1. Relax `(?i)^[\s#/*-]*note:\s*\w` to be more specific (e.g., `note:\s*(changed|modified|updated|added|removed|implemented|refactored)`)
|
||||
2. Add a dedicated `LegitimateCommentFilter` to the filter pipeline in `main.go`
|
||||
3. Support `--allow-prefix` CLI flag for external configuration
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
## Summary
|
||||
|
||||
- Add `allowed_comment_prefixes` config to `CommentCheckerConfigSchema` with sensible defaults (Note:, TODO:, FIXME:, HACK:, WARNING:, etc.)
|
||||
- Add post-processing filter in `allowed-prefix-filter.ts` that suppresses false positives from the Go binary's output before appending to tool output
|
||||
- Add 11 test cases covering false positive scenarios (Note:, TODO:, FIXME:, case-insensitivity, mixed comments, agent memo header suppression)
|
||||
|
||||
## Problem
|
||||
|
||||
The comment-checker hook's upstream Go binary (`go-claude-code-comment-checker`) flags ALL non-filtered comments as problematic. Its `AgentMemoFilter` regex `(?i)^[\s#/*-]*note:\s*\w` classifies any `Note:` comment as AI-generated "agent memo" slop, triggering an aggressive warning banner.
|
||||
|
||||
This causes false positives for legitimate, widely-used comment patterns:
|
||||
```typescript
|
||||
// Note: Thread-safe implementation required due to concurrent access
|
||||
// NOTE: See RFC 7231 section 6.5.4 for 404 semantics
|
||||
// Note: This timeout matches the upstream service SLA
|
||||
```
|
||||
|
||||
These are standard engineering documentation patterns, not AI slop.
|
||||
|
||||
## Solution
|
||||
|
||||
Rather than waiting for an upstream binary fix, this PR adds a configurable **post-processing filter** on the TypeScript side:
|
||||
|
||||
1. **Config**: `comment_checker.allowed_comment_prefixes` - array of case-insensitive prefixes (defaults: `note:`, `todo:`, `fixme:`, `hack:`, `warning:`, `important:`, `bug:`, etc.)
|
||||
2. **Filter**: After the Go binary returns flagged comments, `filterAllowedComments()` parses the XML output and suppresses comments matching allowed prefixes
|
||||
3. **Behavior**: If ALL flagged comments are legitimate → suppress entire warning. If mixed → remove only the legitimate entries from the XML, keep the warning for actual slop.
|
||||
|
||||
Users can customize via config:
|
||||
```jsonc
|
||||
{
|
||||
"comment_checker": {
|
||||
"allowed_comment_prefixes": ["note:", "todo:", "fixme:", "custom-prefix:"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Test Plan
|
||||
|
||||
- 11 new test cases in `allowed-prefix-filter.test.ts`
|
||||
- Updated assertion in `hook.apply-patch.test.ts` for new parameter
|
||||
- `bun test src/hooks/comment-checker/` passes
|
||||
- `bun run typecheck` clean
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
# Verification Strategy
|
||||
|
||||
## 1. Unit Tests
|
||||
|
||||
### New test file: `allowed-prefix-filter.test.ts`
|
||||
Run: `bun test src/hooks/comment-checker/allowed-prefix-filter.test.ts`
|
||||
|
||||
| # | Scenario | Input | Expected |
|
||||
|---|----------|-------|----------|
|
||||
| 1 | Only Note: comments (default prefixes) | `// Note: Thread-safe`, `// NOTE: See RFC` | `hasRemainingComments: false`, empty message |
|
||||
| 2 | Only TODO/FIXME/HACK (default prefixes) | `// TODO: impl`, `// FIXME: race`, `# HACK: workaround` | Suppressed |
|
||||
| 3 | Only AI slop comments | `// Added validation`, `// Refactored for perf` | Full message preserved |
|
||||
| 4 | Mixed legitimate + slop | `// Note: Thread-safe`, `// Changed from old to new` | Message kept, Note: entry removed from XML |
|
||||
| 5 | Case-insensitive Note: | `// note: lowercase test` | Suppressed |
|
||||
| 6 | Hash-prefixed comments | `# Note: Python`, `# TODO: something` | Suppressed (prefix stripped before matching) |
|
||||
| 7 | Security: prefix | `// Security: validate input` | Suppressed |
|
||||
| 8 | Warning: prefix | `// WARNING: mutates input` | Suppressed |
|
||||
| 9 | Empty allowed prefixes | `// Note: should pass through` | Full message preserved (no filtering) |
|
||||
| 10 | Custom prefix | `// PERF: O(n log n)` with `["perf:"]` | Suppressed |
|
||||
| 11 | Agent memo header + Note: | Full agent memo banner + `// Note: Thread-safe` | Entire message suppressed including banner |
|
||||
|
||||
### Existing test: `hook.apply-patch.test.ts`
|
||||
Run: `bun test src/hooks/comment-checker/hook.apply-patch.test.ts`
|
||||
|
||||
Verify the updated mock assertion accepts the new `allowedPrefixes` array parameter.
|
||||
|
||||
### Existing test: `cli.test.ts`
|
||||
Run: `bun test src/hooks/comment-checker/cli.test.ts`
|
||||
|
||||
Verify no regressions in binary spawning, timeout, and semaphore logic.
|
||||
|
||||
## 2. Type Checking
|
||||
|
||||
```bash
|
||||
bun run typecheck
|
||||
```
|
||||
|
||||
Verify:
|
||||
- `CommentCheckerConfigSchema` change propagates correctly to `CommentCheckerConfig` type
|
||||
- All call sites in `hook.ts` and `cli-runner.ts` pass the new parameter
|
||||
- `filterAllowedComments` return type matches usage in `cli-runner.ts`
|
||||
- No new type errors introduced
|
||||
|
||||
## 3. LSP Diagnostics
|
||||
|
||||
```bash
|
||||
# Check all changed files for errors
|
||||
lsp_diagnostics src/config/schema/comment-checker.ts
|
||||
lsp_diagnostics src/hooks/comment-checker/allowed-prefix-filter.ts
|
||||
lsp_diagnostics src/hooks/comment-checker/cli-runner.ts
|
||||
lsp_diagnostics src/hooks/comment-checker/hook.ts
|
||||
lsp_diagnostics src/hooks/comment-checker/allowed-prefix-filter.test.ts
|
||||
```
|
||||
|
||||
## 4. Full Test Suite
|
||||
|
||||
```bash
|
||||
bun test src/hooks/comment-checker/
|
||||
```
|
||||
|
||||
All 4 test files should pass:
|
||||
- `cli.test.ts` (existing - no regressions)
|
||||
- `pending-calls.test.ts` (existing - no regressions)
|
||||
- `hook.apply-patch.test.ts` (modified assertion)
|
||||
- `allowed-prefix-filter.test.ts` (new - all 11 cases)
|
||||
|
||||
## 5. Build Verification
|
||||
|
||||
```bash
|
||||
bun run build
|
||||
```
|
||||
|
||||
Ensure the new module is properly bundled and exported.
|
||||
|
||||
## 6. Integration Verification (Manual)
|
||||
|
||||
If binary is available locally:
|
||||
|
||||
```bash
|
||||
# Test with a file containing Note: comment
|
||||
echo '{"session_id":"test","tool_name":"Write","transcript_path":"","cwd":"/tmp","hook_event_name":"PostToolUse","tool_input":{"file_path":"/tmp/test.ts","content":"// Note: Thread-safe implementation\nconst x = 1"}}' | ~/.cache/oh-my-opencode/bin/comment-checker check
|
||||
echo "Exit code: $?"
|
||||
```
|
||||
|
||||
Expected: Binary returns exit 2 (comment detected), but the TypeScript post-filter should suppress it.
|
||||
|
||||
## 7. Config Validation
|
||||
|
||||
Test that config changes work:
|
||||
|
||||
```jsonc
|
||||
// .opencode/oh-my-opencode.jsonc
|
||||
{
|
||||
"comment_checker": {
|
||||
// Override: only allow Note: and TODO:
|
||||
"allowed_comment_prefixes": ["note:", "todo:"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Verify Zod schema accepts the config and defaults are applied when field is omitted.
|
||||
|
||||
## 8. Regression Checks
|
||||
|
||||
Verify the following still work correctly:
|
||||
- AI slop comments (`// Added new feature`, `// Refactored for performance`) are still flagged
|
||||
- BDD comments (`// given`, `// when`, `// then`) are still allowed (binary-side filter)
|
||||
- Linter directives (`// eslint-disable`, `// @ts-ignore`) are still allowed (binary-side filter)
|
||||
- Shebangs (`#!/usr/bin/env node`) are still allowed (binary-side filter)
|
||||
- `custom_prompt` config still works
|
||||
- Semaphore prevents concurrent comment-checker runs
|
||||
- Timeout handling (30s) still works
|
||||
|
||||
## 9. Edge Cases to Watch
|
||||
|
||||
- Empty message from binary (exit code 0) - filter should be no-op
|
||||
- Binary not available - hook gracefully degrades (existing behavior)
|
||||
- Message with no `<comment>` XML elements - filter passes through
|
||||
- Very long messages with many comments - regex performance
|
||||
- Comments containing XML-special characters (`<`, `>`, `&`) in text
|
||||
@@ -0,0 +1 @@
|
||||
{"total_tokens": null, "duration_ms": 399000, "total_duration_seconds": 399}
|
||||
Reference in New Issue
Block a user